public void ReplaceChunkDatas(List <IntVector4> newChunkPosList, ProcToMergeChunkAtPos mergeChunkDataAtPos)
{
Dictionary <IntVector4, Dictionary <IntVector4, List <IntVector4> > > file2piece2chunkList = new Dictionary <IntVector4, Dictionary <IntVector4, List <IntVector4> > >();
int n = newChunkPosList.Count;
int px, py, pz;
int fx, fz;
for (int i = 0; i < n; i++)
{
IntVector4 chunkPos = newChunkPosList[i];
VFFileUtil.WorldChunkPosToPiecePos(chunkPos, out px, out py, out pz);
VFFileUtil.PiecePos2FilePos(px, py, pz, chunkPos.w, out fx, out fz);
IntVector4 filePos = new IntVector4(fx, 0, fz, chunkPos.w);
IntVector4 piecePos = new IntVector4(px, py, pz, chunkPos.w);
if (!file2piece2chunkList.ContainsKey(filePos))
{
file2piece2chunkList.Add(filePos, new Dictionary <IntVector4, List <IntVector4> >());
}
if (!file2piece2chunkList[filePos].ContainsKey(piecePos))
{
file2piece2chunkList[filePos].Add(piecePos, new List <IntVector4>());
}
file2piece2chunkList[filePos][piecePos].Add(chunkPos);
}
List <IntVector4> fileIndexList = file2piece2chunkList.Keys.Cast <IntVector4>().ToList();
int nFileIndexList = fileIndexList.Count;
for (int i = 0; i < nFileIndexList; i++)
{
IntVector4 fileIndex = fileIndexList[i];
ReplacePiecesInFile(file2piece2chunkList[fileIndex], mergeChunkDataAtPos, fileIndex);
}
}
public void ReadPieceDataToBuff(int px, int py, int pz, int lod) // ret if cache hit
{
int fx, fz;
VFFileUtil.PiecePos2FilePos(px, py, pz, lod, out fx, out fz);
ReadPieceDataToBuff(px, py, pz, lod, fx, fz);
}
public void AddRequest(VFVoxelChunkData chunkData)
{
IntVector4 chunkPos = chunkData.ChunkPosLod;
// Req to chunks out of range will return, these chunks will be null and can not be write according to WriteVoxelAtIdx's code
if (chunkPos.x < 0 || chunkPos.x >= VoxelTerrainConstants._worldMaxCX ||
chunkPos.y < 0 || chunkPos.y >= VoxelTerrainConstants.WorldMaxCY(chunkPos.w) ||
chunkPos.z < 0 || chunkPos.z >= VoxelTerrainConstants._worldMaxCZ)
{
chunkData.OnDataLoaded(VFVoxelChunkData.S_ChunkDataAir);
return;
}
if (_bImmMode) // no caching
{
int px, py, pz;
VFFileUtil.WorldChunkPosToPiecePos(chunkPos, out px, out py, out pz);
ReadPieceDataToBuff(px, py, pz, chunkPos.w);
_buff.Decompress(px, py, pz, chunkPos.w);
_buff.SetChunkData(chunkData, _chunkDataProc);
}
else
{
ReqValue reqValue;
if (!_chunkReqList.TryGetValue(chunkData, out reqValue) || !chunkData.IsStampIdentical(reqValue.stamp))
{
VFFileUtil.WorldChunkPosToPiecePos(chunkPos, out reqValue.px, out reqValue.py, out reqValue.pz);
reqValue.pw = chunkPos.w;
reqValue.stamp = chunkData.StampOfUpdating;
_chunkReqList[chunkData] = reqValue;
}
}
}
private void FetchFile(int fx, int fz, int lod)
{
if (fx != _curFileX || fz != _curFileZ || lod != _curFileLod)
{
if (_fileStreams != null)
{
int idx = lod * VoxelTerrainConstants._mapFileCountX * VoxelTerrainConstants._mapFileCountZ + fz * VoxelTerrainConstants._mapFileCountX + fx;
_curFileStream = _fileStreams [idx];
_curFileLen = _fileLens [idx];
}
else
{
VFFileUtil.PreOpenFile(_dataFilePrefix, fx, fz, lod, out _curFileStream, out _curFileLen);
}
if (_curFileStream != null)
{
int ofsDataLenLod = (VoxelTerrainConstants._mapPieceCountXorZ >> lod) * (VoxelTerrainConstants._mapPieceCountXorZ >> lod) * (VoxelTerrainConstants._mapPieceCountY >> lod) * 4;
_curFileStream.Seek(0, SeekOrigin.Begin);
_curFileStream.Read(_ofsData, 0, ofsDataLenLod);
}
_curFileX = fx;
_curFileZ = fz;
_curFileLod = lod;
}
}
public VFDataReader(string dataFilePrefix, ChunkDataLoadedProcessor chunkDataProc = null, bool bPreopenFiles = true)
{
_dataFilePrefix = dataFilePrefix;
_chunkDataProc = chunkDataProc;
if (bPreopenFiles)
{
VFFileUtil.PreOpenAllFiles(_dataFilePrefix, out _fileStreams, out _fileLens);
}
}
private byte[] _ofsData = new byte[VFFileUtil.OfsDataLen]; // use the max of all lod's ofsdata len
private void GetPieceOfsNLen(long fileLen, int px, int py, int pz, int lod, out int pieceDataOfs, out int pieceDataLen)
{
int idx = VFFileUtil.PiecePos2IndexInFile(px, py, pz, lod) * 4; //in byte
pieceDataOfs = _ofsData[idx] + (_ofsData[idx + 1] << 8) + (_ofsData[idx + 2] << 16) + (_ofsData[idx + 3] << 24);
int fSetOfsDataLenLod = (VoxelTerrainConstants._mapPieceCountXorZ >> lod) * (VoxelTerrainConstants._mapPieceCountXorZ >> lod) * (VoxelTerrainConstants._mapPieceCountY >> lod) * 4;
if (idx >= fSetOfsDataLenLod - 4)
{
pieceDataLen = (int)(fileLen - pieceDataOfs);
}
else
{
pieceDataLen = _ofsData[idx + 4] + (_ofsData[idx + 5] << 8) + (_ofsData[idx + 6] << 16) + (_ofsData[idx + 7] << 24) - pieceDataOfs;
}
}
void decompressData(VFVoxelChunkData chunkData)
{
int lod = chunkData.LOD;
int px, py, pz;
VFFileUtil.WorldChunkPosToPiecePos(chunkData.ChunkPosLod, out px, out py, out pz);
int fx, fz;
VFFileUtil.PiecePos2FilePos(px, py, pz, lod, out fx, out fz);
VFPieceDataClone pieceData = VFDataReaderClone.GetPieceDataSub(new IntVector4(px, py, pz, lod), VFDataReaderClone.GetFileSetSub(new IntVector4(fx, 0, fz, lod)));
pieceData.Decompress();
pieceData.SetChunkData(chunkData);
pieceData._data = null;
pieceData = null;
}
public VFVoxelChunkData ReadChunkImm(IntVector4 cpos) // Read Chunk Data immediately
{
VFVoxelChunkData chunkData = new VFVoxelChunkData(null);
chunkData.ChunkPosLod_w = cpos;
// Req to chunks out of range will return, these chunks will be null and can not be write according to WriteVoxelAtIdx's code
if (cpos.x < 0 || cpos.x >= VoxelTerrainConstants._worldMaxCX ||
cpos.y < 0 || cpos.y >= VoxelTerrainConstants.WorldMaxCY(cpos.w) ||
cpos.z < 0 || cpos.z >= VoxelTerrainConstants._worldMaxCZ)
{
chunkData.OnDataLoaded(VFVoxelChunkData.S_ChunkDataAir);
return(chunkData);
}
int px, py, pz;
VFFileUtil.WorldChunkPosToPiecePos(cpos, out px, out py, out pz);
ReadPieceDataToBuff(px, py, pz, cpos.w);
_buff.Decompress(px, py, pz, cpos.w);
_buff.SetChunkData(chunkData, _chunkDataProc);
return(chunkData);
}
public void SetChunkData(VFVoxelChunkData chunkData, ChunkDataLoadedProcessor chunkDataProc)
{
byte[] chunkDataVT;
//bool bFromPool;
byte vol, typ;
if (IsHollow())
{
vol = zippedDataBuffer[0];
typ = zippedDataBuffer[1];
if (vol == 0)
{
chunkDataVT = VFVoxelChunkData.S_ChunkDataAir;
}
else if (vol == 128)
{
chunkDataVT = VFVoxelChunkData.S_ChunkDataWaterPlane;
}
else if (vol == 255)
{
chunkDataVT = VFVoxelChunkData.S_ChunkDataSolid[typ];
}
else
{
chunkDataVT = new byte[VFVoxel.c_VTSize] {
vol, typ
}
};
if (chunkDataProc == null)
{
chunkData.OnDataLoaded(chunkDataVT, false);
}
else
{
chunkDataProc(chunkData, chunkDataVT, false);
}
return;
}
byte[] chunkDataSet = unzippedDataBuffer;
int lenChunkDataSet = unzippedDataLen;
int ofsDataAddr = 0;
int idxChunk = VFFileUtil.ChunkPos2IndexInPiece(chunkData.ChunkPosLod);
int idChunkMax = VoxelTerrainConstants._ChunksPerPiece - 1;
ofsDataAddr += 4 * idxChunk;
try{
int vxDataAddr = (chunkDataSet[ofsDataAddr]) + (chunkDataSet[ofsDataAddr + 1] << 8) + (chunkDataSet[ofsDataAddr + 2] << 16) + (chunkDataSet[ofsDataAddr + 3] << 24);
int vxDataNextAddr = (idxChunk < idChunkMax) ? (chunkDataSet[ofsDataAddr + 4]) + (chunkDataSet[ofsDataAddr + 5] << 8) + (chunkDataSet[ofsDataAddr + 6] << 16) + (chunkDataSet[ofsDataAddr + 7] << 24)
: lenChunkDataSet;
int dataLen = vxDataNextAddr - vxDataAddr;
if (dataLen == VoxelTerrainConstants.VOXEL_ARRAY_LENGTH_VT)
{
chunkDataVT = VFVoxelChunkData.s_ChunkDataPool.Get();
Array.Copy(chunkDataSet, vxDataAddr, chunkDataVT, 0, VoxelTerrainConstants.VOXEL_ARRAY_LENGTH_VT);
if (chunkDataProc == null)
{
chunkData.OnDataLoaded(chunkDataVT, true);
}
else
{
chunkDataProc(chunkData, chunkDataVT, true);
}
}
else if (dataLen == VFVoxel.c_VTSize)
{
vol = chunkDataSet[vxDataAddr];
typ = chunkDataSet[vxDataAddr + 1];
if (vol == 0)
{
chunkDataVT = VFVoxelChunkData.S_ChunkDataAir;
}
else if (vol == 128)
{
chunkDataVT = VFVoxelChunkData.S_ChunkDataWaterPlane;
}
else if (vol == 255)
{
chunkDataVT = VFVoxelChunkData.S_ChunkDataSolid[typ];
}
else
{
chunkDataVT = new byte[VFVoxel.c_VTSize] {
vol, typ
}
};
if (chunkDataProc == null)
{
chunkData.OnDataLoaded(chunkDataVT, false);
}
else
{
chunkDataProc(chunkData, chunkDataVT, false);
}
}
else
{
Debug.LogWarning("[VFDATAReader]Unsupported data length(" + dataLen + ")@" + chunkData.ChunkPosLod);
}
} catch {
Debug.LogWarning("[VFDATAReader]Failed to read Chunk" + chunkData.ChunkPosLod);
}
}
}
// Output : _buff.zippedDataBuffer, _buff.zippedDataLen
public void ReplaceChunksInPiece(List <IntVector4> chunkPosList, ProcToMergeChunkAtPos mergeChunkDataAtPos, IntVector4 piecePos)
{
ReadPieceDataToBuff(piecePos.x, piecePos.y, piecePos.z, piecePos.w);
_buff.Decompress(piecePos.x, piecePos.y, piecePos.z, piecePos.w);
List <IntVector4> sortedChunkPosList = chunkPosList.OrderBy(pos => VFFileUtil.ChunkPos2IndexInPiece(pos)).ToList();
byte[] chunkDataSet = _buff.unzippedDataBuffer;
int lenChunkDataSet = _buff.unzippedDataLen;
byte[] hollowData = null;
if (_buff.IsHollow())
{
hollowData = new byte[_buff.zippedDataLen];
Array.Copy(_buff.zippedDataBuffer, hollowData, _buff.zippedDataLen);
}
int idxChunkMax = VoxelTerrainConstants._ChunksPerPiece - 1;
int idxInPiece = 0;
List <byte[]> chunkDataList = new List <byte[]>();
byte[] chunkData = null;
int sumDataLen = 0;
int n = sortedChunkPosList.Count;
for (int i = 0; i < n; i++)
{
IntVector4 chunkPos = sortedChunkPosList[i];
int idxChunk = VFFileUtil.ChunkPos2IndexInPiece(chunkPos);
while (idxInPiece <= idxChunk)
{
if (_buff.IsHollow())
{
chunkDataList.Add(hollowData);
sumDataLen += hollowData.Length;
}
else
{
int ofsDataAddr = 4 * idxInPiece;
int vxDataAddr = (chunkDataSet[ofsDataAddr]) + (chunkDataSet[ofsDataAddr + 1] << 8) + (chunkDataSet[ofsDataAddr + 2] << 16) + (chunkDataSet[ofsDataAddr + 3] << 24);
int vxDataNextAddr = (idxInPiece < idxChunkMax) ? (chunkDataSet[ofsDataAddr + 4]) + (chunkDataSet[ofsDataAddr + 5] << 8) + (chunkDataSet[ofsDataAddr + 6] << 16) + (chunkDataSet[ofsDataAddr + 7] << 24)
: lenChunkDataSet;
int len = vxDataNextAddr - vxDataAddr;
chunkData = new byte[len];
Array.Copy(chunkDataSet, vxDataAddr, chunkData, 0, len);
chunkDataList.Add(chunkData);
sumDataLen += len;
}
idxInPiece++;
}
// merge chunk
sumDataLen -= chunkDataList[idxChunk].Length;
chunkDataList[idxChunk] = mergeChunkDataAtPos(chunkPos, chunkDataList[idxChunk]);
sumDataLen += chunkDataList[idxChunk].Length;
}
while (idxInPiece <= idxChunkMax)
{
if (_buff.IsHollow())
{
chunkDataList.Add(hollowData);
sumDataLen += hollowData.Length;
}
else
{
int ofsDataAddr = 4 * idxInPiece;
int vxDataAddr = (chunkDataSet[ofsDataAddr]) + (chunkDataSet[ofsDataAddr + 1] << 8) + (chunkDataSet[ofsDataAddr + 2] << 16) + (chunkDataSet[ofsDataAddr + 3] << 24);
int vxDataNextAddr = (idxInPiece < idxChunkMax) ? (chunkDataSet[ofsDataAddr + 4]) + (chunkDataSet[ofsDataAddr + 5] << 8) + (chunkDataSet[ofsDataAddr + 6] << 16) + (chunkDataSet[ofsDataAddr + 7] << 24)
: lenChunkDataSet;
int len = vxDataNextAddr - vxDataAddr;
chunkData = new byte[len];
Array.Copy(chunkDataSet, vxDataAddr, chunkData, 0, len);
chunkDataList.Add(chunkData);
sumDataLen += len;
}
idxInPiece++;
}
int ofsDataLen = 4 * VoxelTerrainConstants._ChunksPerPiece;
_buff.unzippedDataLen = ofsDataLen + sumDataLen;
byte[] pieceData = _buff.unzippedDataBuffer;
int ofs = ofsDataLen;
for (int i = 0; i < VoxelTerrainConstants._ChunksPerPiece; i++)
{
byte[] data = chunkDataList[i];
Array.Copy(data, 0, pieceData, ofs, data.Length);
pieceData[i * 4] = (byte)(ofs & 0xff);
pieceData[i * 4 + 1] = (byte)((ofs >> 8) & 0xff);
pieceData[i * 4 + 2] = (byte)((ofs >> 16) & 0xff);
pieceData[i * 4 + 3] = (byte)((ofs >> 24) & 0xff);
ofs += data.Length;
}
//Compress
_buff.zippedDataLen = LZ4.LZ4_compress(_buff.unzippedDataBuffer, _buff.zippedDataBuffer, _buff.unzippedDataLen);
}
public void ReplacePiecesInFile(Dictionary <IntVector4, List <IntVector4> > piece2chunkList, ProcToMergeChunkAtPos mergeChunkDataAtPos, IntVector4 fileIndex)
{
int lod = fileIndex.w;
string newFileName = _dataFilePrefix + "_x" + fileIndex.x + "_y" + fileIndex.z + (lod != 0 ? ("_" + lod + ".tmp") : ".tmp");
string fileName = _dataFilePrefix + "_x" + fileIndex.x + "_y" + fileIndex.z + (lod != 0 ? ("_" + lod + ".voxelform") : ".voxelform");
using (FileStream fsOld = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read))
{
using (FileStream fsNew = new FileStream(newFileName, FileMode.Create))
{
int nMaxPieceCount = (VoxelTerrainConstants._mapPieceCountXorZ >> lod) * (VoxelTerrainConstants._mapPieceCountXorZ >> lod) * (VoxelTerrainConstants._mapPieceCountY >> lod);
int fSetOfsDataLenLod = nMaxPieceCount * 4;
byte[] ofsDataOld = new byte[fSetOfsDataLenLod];
fsOld.Read(ofsDataOld, 0, fSetOfsDataLenLod);
byte[] ofsDataNew = new byte[fSetOfsDataLenLod];
fsNew.Seek(fSetOfsDataLenLod, SeekOrigin.Begin);
List <IntVector4> piecePosList = piece2chunkList.Keys.Cast <IntVector4>().ToList();
List <IntVector4> sortedPiecePosList = piecePosList.OrderBy(p => VFFileUtil.PiecePos2IndexInFile(p.x, p.y, p.z, p.w)).ToList();
int idx = 0;
int idxInFile = 0;
int pieceDataOfs = 0;
int pieceDataLen = 0;
int curFilePos = 0;
byte[] tmpBuff = new byte[(int)((VoxelTerrainConstants.VOXEL_NUM_PER_PIECE * VFVoxel.c_VTSize + 256) * 1.004f)];
int n = sortedPiecePosList.Count;
for (int i = 0; i < n; i++)
{
IntVector4 piecePos = sortedPiecePosList[i];
int idxPiece = VFFileUtil.PiecePos2IndexInFile(piecePos.x, piecePos.y, piecePos.z, piecePos.w);
while (idxInFile < idxPiece)
{
idx = idxInFile * 4;
pieceDataOfs = ofsDataOld[idx] + (ofsDataOld[idx + 1] << 8) + (ofsDataOld[idx + 2] << 16) + (ofsDataOld[idx + 3] << 24);
if (idx >= fSetOfsDataLenLod - 4)
{
pieceDataLen = (int)fsOld.Length - pieceDataOfs;
}
else
{
pieceDataLen = ofsDataOld[idx + 4] + (ofsDataOld[idx + 5] << 8) + (ofsDataOld[idx + 6] << 16) + (ofsDataOld[idx + 7] << 24) - pieceDataOfs;
}
fsOld.Seek(pieceDataOfs, SeekOrigin.Begin);
fsOld.Read(tmpBuff, 0, pieceDataLen);
curFilePos = (int)fsNew.Position;
ofsDataNew[idx] = (byte)(curFilePos & 0xff);
ofsDataNew[idx + 1] = (byte)((curFilePos >> 8) & 0xff);
ofsDataNew[idx + 2] = (byte)((curFilePos >> 16) & 0xff);
ofsDataNew[idx + 3] = (byte)((curFilePos >> 24) & 0xff);
fsNew.Write(tmpBuff, 0, pieceDataLen);
idxInFile++;
}
ReplaceChunksInPiece(piece2chunkList[piecePos], mergeChunkDataAtPos, piecePos);
curFilePos = (int)fsNew.Position;
idx = idxPiece * 4;
ofsDataNew[idx] = (byte)(curFilePos & 0xff);
ofsDataNew[idx + 1] = (byte)((curFilePos >> 8) & 0xff);
ofsDataNew[idx + 2] = (byte)((curFilePos >> 16) & 0xff);
ofsDataNew[idx + 3] = (byte)((curFilePos >> 24) & 0xff);
fsNew.Write(_buff.zippedDataBuffer, 0, _buff.zippedDataLen);
idxInFile++;
}
while (idxInFile < nMaxPieceCount)
{
idx = idxInFile * 4;
pieceDataOfs = ofsDataOld[idx] + (ofsDataOld[idx + 1] << 8) + (ofsDataOld[idx + 2] << 16) + (ofsDataOld[idx + 3] << 24);
if (idx >= fSetOfsDataLenLod - 4)
{
pieceDataLen = (int)fsOld.Length - pieceDataOfs;
}
else
{
pieceDataLen = ofsDataOld[idx + 4] + (ofsDataOld[idx + 5] << 8) + (ofsDataOld[idx + 6] << 16) + (ofsDataOld[idx + 7] << 24) - pieceDataOfs;
}
fsOld.Seek(pieceDataOfs, SeekOrigin.Begin);
fsOld.Read(tmpBuff, 0, pieceDataLen);
curFilePos = (int)fsNew.Position;
ofsDataNew[idx] = (byte)(curFilePos & 0xff);
ofsDataNew[idx + 1] = (byte)((curFilePos >> 8) & 0xff);
ofsDataNew[idx + 2] = (byte)((curFilePos >> 16) & 0xff);
ofsDataNew[idx + 3] = (byte)((curFilePos >> 24) & 0xff);
fsNew.Write(tmpBuff, 0, pieceDataLen);
idxInFile++;
}
fsNew.Seek(0, SeekOrigin.Begin);
fsNew.Write(ofsDataNew, 0, fSetOfsDataLenLod);
}
}
VFFileUtil.CloseAllFiles(ref _curFileStream, ref _fileStreams, ref _fileLens);
string oldFileName = string.Format(fileName + ".{0:MMdd_hhmmss}", DateTime.Now);
File.Move(fileName, oldFileName);
File.Move(newFileName, fileName);
}
public void Close()
{
VFFileUtil.CloseAllFiles(ref _curFileStream, ref _fileStreams, ref _fileLens);
}
