public void AddVertex(ByteCoord vert)
{
coords.Add(vert);
}
void LateUpdate()
{
if (!Application.isPlaying || !EditorApplication.isPlayingOrWillChangePlaymode || !startRecord)
{
return;
}
currentTime += Time.deltaTime;
if (currentTime > frameInterval)
{
currentTime -= frameInterval;
}
else
{
return;
}
bool isIframe = frameCnt == 0 ? true : false;
if (isIframe)
{
tiling.SetInt("packSize", packageSize);
tiling.SetInt("range", containerSize);
}
if (renderers != null)
{
Dictionary <int, TilePacker> dicPacks = new Dictionary <int, TilePacker>();
List <FragmentVertex[]> frag = new List <FragmentVertex[]>();
for (int i = 0; i < renderers.Count; i++)
{
SkinnedMeshRenderer smr = renderers[i];
if (smr != null)
{
//convert to tiled vertices
Mesh tempMesh = new Mesh();
smr.BakeMesh(tempMesh);
//calculate shader threads and dipatch size;
int alignedNum = 16;
string kernelName = "cs_main8";
int verts = tempMesh.vertices.Length;
int dispatch = 2;
for (int k = 0; k < alignedVerts.Length; k++)
{
int aligned = alignedVerts[k];
if (verts - aligned <= 0)
{
alignedNum = aligned;
kernelName = kernelNames[k];
dispatch = dispatches[k];
break;
}
}
Vector3[] src = new Vector3[alignedNum];
tempMesh.vertices.CopyTo(src, 0);
DestroyImmediate(tempMesh);
Quaternion quat = smr.transform.rotation;
Vector3 pos = smr.transform.position - targetGameObject.transform.position;
Vector3 scale = new Vector3(1, 1, 1);
Matrix4x4 wrd = Matrix4x4.TRS(pos, quat, scale);
float[] wrdMatrix =
{
wrd.m00, wrd.m01, wrd.m02, wrd.m03,
wrd.m10, wrd.m11, wrd.m12, wrd.m13,
wrd.m20, wrd.m21, wrd.m22, wrd.m23,
wrd.m30, wrd.m31, wrd.m32, wrd.m33
};
if (isIframe)
{
tiling.SetFloats("wrdMatrix", wrdMatrix);
tiling.SetInt("modelGroup", i);
ComputeBuffer srcBuf = new ComputeBuffer(alignedNum, Marshal.SizeOf(typeof(Vector3)));
ComputeBuffer destBuf = new ComputeBuffer(alignedNum, Marshal.SizeOf(typeof(TiledVertex)));
srcBuf.SetData(src);
int kernelNum = tiling.FindKernel(kernelName);
tiling.SetBuffer(kernelNum, "srcBuf", srcBuf);
tiling.SetBuffer(kernelNum, "destBuf", destBuf);
tiling.Dispatch(kernelNum, dispatch, 1, 1);
TiledVertex[] data = new TiledVertex[src.Length];
destBuf.GetData(data);
srcBuf.Release();
destBuf.Release();
for (int j = 0; j < verts; j++)
{
TiledVertex vert = data[j];
int tx = (vert.tileID & 0xFF);
int ty = (vert.tileID & 0xFF00) >> 8;
int tz = (vert.tileID & 0xFF0000) >> 16;
int tileID = tx + ty * packageSize + tz * (packageSize * packageSize);
if (tx == 255 && ty == 255 && tz == 255)
{
continue;
}
TilePacker tile;
if (!dicPacks.TryGetValue(tileID, out tile))
{
tile = new TilePacker(tx, ty, tz);
dicPacks.Add(tileID, tile);
}
ByteCoord coord = new ByteCoord();
coord.p1 = (byte)((vert.polyIndex & 0xFF));
coord.p2 = (byte)((vert.polyIndex & 0xFF00) >> 8);
coord.p3 = (byte)((vert.polyIndex & 0xFF0000) >> 16);
coord.x = (byte)vert.x;
coord.y = (byte)vert.y;
coord.z = (byte)vert.z;
dicPacks[tileID].AddVertex(coord);
}
}
else
{
diff.SetFloats("wrdMatrix", wrdMatrix);
diff.SetFloats("oldMatrix", oldMatrix[i]);
ComputeBuffer srcBuf = new ComputeBuffer(alignedNum, Marshal.SizeOf(typeof(Vector3)));
ComputeBuffer oldBuf = new ComputeBuffer(alignedNum, Marshal.SizeOf(typeof(Vector3)));
ComputeBuffer destBuf = new ComputeBuffer(alignedNum, Marshal.SizeOf(typeof(FragmentVertex)));
srcBuf.SetData(src);
oldBuf.SetData(oldVertsBuf[i]);
int kernelNum = diff.FindKernel(kernelName);
diff.SetBuffer(kernelNum, "srcBuf", srcBuf);
diff.SetBuffer(kernelNum, "oldBuf", oldBuf);
diff.SetBuffer(kernelNum, "destBuf", destBuf);
diff.Dispatch(kernelNum, dispatch, 1, 1);
FragmentVertex[] data = new FragmentVertex[src.Length];
destBuf.GetData(data);
srcBuf.Release();
oldBuf.Release();
destBuf.Release();
frag.Add(data);
}
oldVertsBuf[i] = src;
oldMatrix[i] = wrdMatrix;
}
}
int packages = 0;
List <byte> lPacks = new List <byte>();
if (isIframe)
{
linedIndices.Clear();
foreach (KeyValuePair <int, TilePacker> p in dicPacks)
{
packages++;
TilePacker pack = p.Value;
lPacks.AddRange(pack.PackToByteArray(packageSize));
linedIndices.AddRange(pack.getIndices());
}
}
else
{
for (int i = 0; i < linedIndices.Count; i++)
{
int meshIndex = (linedIndices[i] >> 16) & 0xFF;
int vertIndex = linedIndices[i] & 0xFFFF;
FragmentVertex f = frag[meshIndex][vertIndex];
byte[] val = new byte[3];
val[0] = (byte)(f.x);
val[1] = (byte)(f.y);
val[2] = (byte)(f.z);
lPacks.AddRange(val);
}
}
int currentBinaryLength = combinedBinary.Count;
if (currentBinaryLength == 0)
{
tempStartTicks = DateTime.Now.Ticks;
}
byte[] streamData =
AddHeader(lPacks.ToArray(), targetGameObject.transform.position, packages, isIframe, timeStamp);
byteSize.Add(streamData.Length);
combinedBinary.AddRange(streamData);
timeStamp++;
if (timeStamp % combinedFrames == 0 && timeStamp > 0)
{
long tick = timeStamp / combinedFrames - 1;
long startTicks = tempStartTicks - recordStartTicks;
long endTicks = DateTime.Now.Ticks - recordStartTicks;
StreamInfo streamInfo = serializer.CreateStreamInfo(tick, startTicks, endTicks);
serializer.Send(streamInfo, tick);
string fileName = tick.ToString("000000") + ".stmv";
byte[] buffer = new byte[(byteSize.Count + 1) * sizeof(int) + combinedBinary.Count];
byteSize.Insert(0, byteSize.Count);
Buffer.BlockCopy(byteSize.ToArray(), 0, buffer, 0, byteSize.Count * sizeof(int));
Buffer.BlockCopy(combinedBinary.ToArray(), 0, buffer, byteSize.Count * sizeof(int), combinedBinary.Count);
//serializer.Send(ExternalTools.Compress(combinedBinary.ToArray()), "stream", fileName);
serializer.Send(Lib.ExternalTools.Compress(buffer), "stream", fileName);
byteSize.Clear();
combinedBinary.Clear();
}
}
frameCnt = (frameCnt + 1) % (subframesPerKeyframe + 1);
}
public void VerticesReceived(byte[] data)
{
if (isRequestComplete)
{
byte[] res = { 0x31, 0x00, 0x00, 0x00, 0x00 };
webRtcManager.RequestData(res, false);
lastDataType = data[0];
if (_interpolate)
{
for (int i = 0; i < vertsBuf.Length; i++)
{
vertsBuf[i].CopyTo(vertsBuf_old[i], 0);
}
position_old = position;
}
//Debug.Log("Received bytes: " + data.Length);
//byte[] stampBuf = { data[1], data[2], data[3], data[4] };
//uint stamp = BitConverter.ToUInt32(stampBuf, 0);
int packages = data[7] * 65536 + data[6] * 256 + data[5];
bool isCompressed = data[8] == 0x01 ? true : false;
byte[][] byteVec = new byte[3][];
for (int i = 0; i < 3; i++)
{
byteVec[i] = new byte[4];
for (int j = 0; j < 4; j++)
{
byteVec[i][j] = data[i * 4 + j + 9];
}
}
position = new Vector3(
BitConverter.ToSingle(byteVec[0], 0),
BitConverter.ToSingle(byteVec[1], 0),
BitConverter.ToSingle(byteVec[2], 0)
);
int offset = 21;
byte[] buf = null;
#if UNITY_WEBGL || BROTLI_NO_COMPRESS
buf = data;
#else
if (isCompressed)
{
int bufSize = data.Length - 21;
byte[] rawbuf = new byte[bufSize];
Buffer.BlockCopy(data, 21, rawbuf, 0, bufSize);
if (!brotli.decompressBuffer(rawbuf, ref buf))
{
Debug.Log("decompress failed!");
return;
}
offset = 0;
}
else
{
buf = data;
}
#endif
if (data[0] == 0x0F)
{
linedIndices.Clear();
for (int i = 0; i < packages; i++)
{
VertexPack vp = new VertexPack();
vp.tx = buf[offset];
vp.ty = buf[offset + 1];
vp.tz = buf[offset + 2];
vp.poly1 = buf[offset + 3];
vp.poly2 = buf[offset + 4];
vp.poly3 = buf[offset + 5];
offset += 6;
int hk = packageSize / 2;
int qk = hk / areaRange;
int vertCount = vp.poly3 * 65536 + vp.poly2 * 256 + vp.poly1;
for (int j = 0; j < vertCount; j++)
{
ByteCoord v = new ByteCoord();
v.p1 = buf[offset + j * 5];
v.p2 = buf[offset + j * 5 + 1];
v.p3 = buf[offset + j * 5 + 2];
int compress = 0;
compress += buf[offset + j * 5 + 3];
compress += (ushort)(buf[offset + j * 5 + 4] << 8);
v.x = (byte)(compress & 0x1F);
v.y = (byte)((compress >> 5) & 0x1F);
v.z = (byte)((compress >> 10) & 0x1F);
float x = ((int)vp.tx - hk) / (float)qk;
float y = ((int)vp.ty - hk) / (float)qk;
float z = ((int)vp.tz - hk) / (float)qk;
x += (float)v.x / (32 * (float)qk);
y += (float)v.y / (32 * (float)qk);
z += (float)v.z / (32 * (float)qk);
int vertIdx = v.p2 * 256 + v.p1;
int meshIdx = v.p3;
Vector3 vert = new Vector3(x, y, z);
vertsBuf[meshIdx][vertIdx] = vert;
linedIndices.Add(meshIdx * 0x10000 + vertIdx);
}
offset += (vertCount * 5);
}
if (getLag && _interpolate)
{
for (int i = 0; i < vertsBuf.Length; i++)
{
vertsBuf[i].CopyTo(vertsBuf_old[i], 0);
}
position_old = position;
}
getLag = false;
_interpolate = interpolateVertices;
}
else if (data[0] == 0x0E && !getLag)
{
for (int i = 0; i < linedIndices.Count; i++)
{
int meshIdx = (linedIndices[i] >> 16) & 0xFF;
int vertIdx = linedIndices[i] & 0xFFFF;
int ix = buf[i * 3 + offset];
int iy = buf[i * 3 + offset + 1];
int iz = buf[i * 3 + offset + 2];
float dx = ((float)ix - 128f) / 128f;
float dy = ((float)iy - 128f) / 128f;
float dz = ((float)iz - 128f) / 128f;
float x = Mathf.Sign(dx) * Mathf.Pow(Mathf.Abs(dx), 2f);
float y = Mathf.Sign(dy) * Mathf.Pow(Mathf.Abs(dy), 2f);
float z = Mathf.Sign(dz) * Mathf.Pow(Mathf.Abs(dz), 2f);
Vector3 vec = vertsBuf[meshIdx][vertIdx];
vec = vec + new Vector3(x, y, z);
vertsBuf[meshIdx][vertIdx] = vec;
}
}
//currentTimeStamp = stamp;
}
if (!_interpolate)
{
if (lagTime >= 0.1f && lastDataType == 0x0E)
{
getLag = true;
UpdateVerts();
return;
}
lagTime = 0.0f;
UpdateVerts();
}
else
{
timeWeight = 0.0f;
}
}
void LateUpdate()
{
if (!isConnected || webRtcManager.connections == 0)
{
return;
}
currentTime += Time.deltaTime;
if (currentTime > frameInterval)
{
currentTime -= frameInterval;
}
else
{
return;
}
bool isIframe = frameCnt == 0 ? true : false;
if (isIframe)
{
tiling.SetInt("packSize", packageSize);
tiling.SetInt("range", areaRange);
}
if (renderers != null)
{
Dictionary <int, TilePacker> dicPacks = new Dictionary <int, TilePacker>();
List <FragmentVertex[]> frag = new List <FragmentVertex[]>();
for (int i = 0; i < renderers.Length; i++)
{
SkinnedMeshRenderer smr = renderers[i];
if (smr != null)
{
//convert to tiled vertices
Mesh tempMesh = new Mesh();
smr.BakeMesh(tempMesh);
//calculate shader threads and dipatch size;
int alignedNum = 16;
string kernelName = "cs_main8";
int verts = tempMesh.vertices.Length;
int dispatch = 2;
for (int k = 0; k < alignedVerts.Length; k++)
{
int aligned = alignedVerts[k];
if (verts - aligned <= 0)
{
alignedNum = aligned;
kernelName = kernelNames[k];
dispatch = dispatches[k];
break;
}
}
Vector3[] src = new Vector3[alignedNum];
tempMesh.vertices.CopyTo(src, 0);
DestroyImmediate(tempMesh);
Quaternion quat = smr.transform.rotation;
Vector3 pos = smr.transform.position - targetGameObject.transform.position;
Vector3 scale = new Vector3(1, 1, 1);
Matrix4x4 wrd = Matrix4x4.TRS(pos, quat, scale);
float[] wrdMatrix =
{
wrd.m00, wrd.m01, wrd.m02, wrd.m03,
wrd.m10, wrd.m11, wrd.m12, wrd.m13,
wrd.m20, wrd.m21, wrd.m22, wrd.m23,
wrd.m30, wrd.m31, wrd.m32, wrd.m33
};
if (isIframe)
{
tiling.SetFloats("wrdMatrix", wrdMatrix);
tiling.SetInt("modelGroup", i);
ComputeBuffer srcBuf = new ComputeBuffer(
alignedNum, Marshal.SizeOf(typeof(Vector3)));
ComputeBuffer destBuf = new ComputeBuffer(
alignedNum, Marshal.SizeOf(typeof(TiledVertex)));
srcBuf.SetData(src);
int kernelNum = tiling.FindKernel(kernelName);
tiling.SetBuffer(kernelNum, "srcBuf", srcBuf);
tiling.SetBuffer(kernelNum, "destBuf", destBuf);
tiling.Dispatch(kernelNum, dispatch, 1, 1);
TiledVertex[] data = new TiledVertex[src.Length];
destBuf.GetData(data);
srcBuf.Release();
destBuf.Release();
for (int j = 0; j < verts; j++)
{
TiledVertex vert = data[j];
int tx = (vert.tileID & 0xFF);
int ty = (vert.tileID & 0xFF00) >> 8;
int tz = (vert.tileID & 0xFF0000) >> 16;
int tileID = tx + ty * packageSize + tz * (packageSize * packageSize);
if (tx == 255 && ty == 255 && tz == 255)
{
continue;
}
TilePacker tile;
if (!dicPacks.TryGetValue(tileID, out tile))
{
tile = new TilePacker(tx, ty, tz);
dicPacks.Add(tileID, tile);
}
ByteCoord coord = new ByteCoord();
coord.p1 = (byte)((vert.polyIndex & 0xFF));
coord.p2 = (byte)((vert.polyIndex & 0xFF00) >> 8);
coord.p3 = (byte)((vert.polyIndex & 0xFF0000) >> 16);
coord.x = (byte)vert.x;
coord.y = (byte)vert.y;
coord.z = (byte)vert.z;
dicPacks[tileID].AddVertex(coord);
}
}
else
{
diff.SetFloats("wrdMatrix", wrdMatrix);
diff.SetFloats("oldMatrix", oldMatrix[i]);
ComputeBuffer srcBuf = new ComputeBuffer(
alignedNum, Marshal.SizeOf(typeof(Vector3)));
ComputeBuffer oldBuf = new ComputeBuffer(
alignedNum, Marshal.SizeOf(typeof(Vector3)));
ComputeBuffer destBuf = new ComputeBuffer(
alignedNum, Marshal.SizeOf(typeof(FragmentVertex)));
srcBuf.SetData(src);
oldBuf.SetData(oldVertsBuf[i]);
int kernelNum = diff.FindKernel(kernelName);
diff.SetBuffer(kernelNum, "srcBuf", srcBuf);
diff.SetBuffer(kernelNum, "oldBuf", oldBuf);
diff.SetBuffer(kernelNum, "destBuf", destBuf);
diff.Dispatch(kernelNum, dispatch, 1, 1);
FragmentVertex[] data = new FragmentVertex[src.Length];
destBuf.GetData(data);
srcBuf.Release();
oldBuf.Release();
destBuf.Release();
frag.Add(data);
}
oldVertsBuf[i] = src;
oldMatrix[i] = wrdMatrix;
}
}
int byteCnt = 0;
int packages = 0;
List <byte> lPacks = new List <byte>();
if (isIframe)
{
linedIndices.Clear();
foreach (KeyValuePair <int, TilePacker> p in dicPacks)
{
packages++;
TilePacker pack = p.Value;
lPacks.AddRange(pack.PackToByteArray(packageSize));
linedIndices.AddRange(pack.getIndices());
}
}
else
{
for (int i = 0; i < linedIndices.Count; i++)
{
int meshIndex = (linedIndices[i] >> 16) & 0xFF;
int vertIndex = linedIndices[i] & 0xFFFF;
FragmentVertex f = frag[meshIndex][vertIndex];
byte[] val = new byte[3];
val[0] = (byte)(f.x);
val[1] = (byte)(f.y);
val[2] = (byte)(f.z);
lPacks.AddRange(val);
}
}
byteCnt += lPacks.Count;
byte[] rawBuf = lPacks.ToArray();
byte[] compressedBuf = null;
int[] prog = new int[1];
#if UNITY_WEBGL || BROTLI_NO_COMPRESS
#else
if (!brotli.compressBuffer(lPacks.ToArray(), ref compressedBuf, prog, quality: 8))
{
Debug.Log("compress failed!");
return;
}
#endif
webRtcManager.SendVertexStream(rawBuf, compressedBuf,
targetGameObject.transform.position, packages, isIframe, timeStamp, true);
timeStamp++;
}
frameCnt = (frameCnt + 1) % (subframesPerKeyframe + 1);
}
