public IEnumerator Prepare(Material azureKinectScreenMaterial, VideoSenderMessage videoMessage)
{
if (State != PrepareState.Unprepared)
{
throw new Exception("State has to be Unprepared to prepare TextureGroupUpdater.");
}
State = PrepareState.Preparing;
textureSet.SetWidth(videoMessage.width);
textureSet.SetHeight(videoMessage.height);
TelepresenceToolkitPlugin.InitTextureGroup(textureSet.GetId());
colorDecoder = new Vp8Decoder();
depthDecoder = new TrvlDecoder(videoMessage.width * videoMessage.height);
State = PrepareState.Prepared;
while (!textureSet.IsInitialized())
{
yield return(null);
}
// TextureGroup includes Y, U, V, and a depth texture.
azureKinectScreenMaterial.SetTexture("_YTex", textureSet.GetYTexture());
azureKinectScreenMaterial.SetTexture("_UvTex", textureSet.GetUvTexture());
azureKinectScreenMaterial.SetTexture("_DepthTex", textureSet.GetDepthTexture());
State = PrepareState.Prepared;
}
// Since calculation including Unproject() takes too much time, this function is made to run as a coroutine.
public IEnumerator Prepare(VideoSenderMessage videoMessage)
{
State = PrepareState.Preparing;
Progress = 0.0f;
int width = videoMessage.width;
int height = videoMessage.height;
var vertices = new Vector3[width * height];
var uv = new Vector2[width * height];
for (int i = 0; i < width; ++i)
{
for (int j = 0; j < height; ++j)
{
float[] xy = new float[2];
int valid = 0;
// TODO: Check whether using videoMessageData.intrinsics.maxRadiusForProjection is correct.
if (KinectSolver.Unproject(videoMessage.intrinsics,
videoMessage.intrinsics.maxRadiusForProjection,
new float[2] {
i, j
}, ref xy, ref valid))
{
// Flip y since Azure Kinect's y axis is downwards.
// https://docs.microsoft.com/en-us/azure/kinect-dk/coordinate-systems
vertices[i + j * width] = new Vector3(xy[0], -xy[1], 1.0f);
}
else
{
vertices[i + j * width] = new Vector3(0.0f, 0.0f, 0.0f);
}
uv[i + j * width] = new Vector2(i / (float)(width - 1), j / (float)(height - 1));
}
Progress = i / (float)width * 0.99f;
yield return(null);
}
Progress = 0.99f;
//print($"vertices[0]: {vertices[0]}"); // (-1.0, 1.0, 1.0): left-top
//print($"vertices[last]: {vertices[vertices.Length - 1]}"); // (0.8, -0.6, 1.0): right-bottom
const float SIZE_AMPLIFIER = 1.2f;
int quadWidth = width - 2;
int quadHeight = height - 2;
var quadVertices = new Vector3[quadWidth * quadHeight];
var quadUv = new Vector2[quadWidth * quadHeight];
var quadHalfSizes = new Vector2[quadWidth * quadHeight];
for (int ii = 0; ii < quadWidth; ++ii)
{
for (int jj = 0; jj < quadHeight; ++jj)
{
int i = ii + 1;
int j = jj + 1;
quadVertices[ii + jj * quadWidth] = vertices[i + j * width];
quadUv[ii + jj * quadWidth] = uv[i + j * width];
// Trying to make both x and y to have a positive number. The first 0.5f is to make the size relevant to
// the vertex in (i, j). The second one is to get the half size of it.
quadHalfSizes[ii + jj * quadWidth] = (vertices[(i + 1) + (j - 1) * width] - vertices[(i - 1) + (j + 1) * width]) * 0.5f * 0.5f * SIZE_AMPLIFIER;
}
}
//print($"quadSizes[0]: {quadSizes[0].x}, {quadSizes[0].y}"); // 0.002900749, 0.003067017
var triangles = new int[quadWidth * quadHeight];
for (int i = 0; i < quadWidth * quadHeight; ++i)
{
triangles[i] = i;
}
// 65.535 is equivalent to (2^16 - 1) / 1000, where (2^16 - 1) is to complement
// the conversion happened in the texture-level from 0 ~ (2^16 - 1) to 0 ~ 1.
// 1000 is the conversion of mm (the unit of Azure Kinect) to m (the unit of Unity3D).
for (int i = 0; i < quadVertices.Length; ++i)
{
quadVertices[i] *= 65.535f;
}
for (int i = 0; i < quadHalfSizes.Length; ++i)
{
quadHalfSizes[i] *= 65.535f;
}
// Without the bounds, Unity decides whether to render this mesh or not based on the vertices calculated here.
// This causes Unity not rendering the mesh transformed by the depth texture even when the transformed one
// belongs to the viewport of the camera.
var bounds = new Bounds(Vector3.zero, Vector3.one * 1000.0f);
var mesh = new Mesh()
{
indexFormat = IndexFormat.UInt32,
vertices = quadVertices,
uv = quadUv,
uv2 = quadHalfSizes,
bounds = bounds,
};
mesh.SetIndices(triangles, MeshTopology.Points, 0);
meshFilter.mesh = mesh;
State = PrepareState.Prepared;
}
public void Assemble(UdpSocket udpSocket,
List <VideoSenderPacket> videoPacketDataList,
List <ParitySenderPacket> parityPacketDataList,
int lastVideoFrameId,
IDictionary <int, VideoSenderMessage> videoMessages)
{
int?addedFrameId = null;
// Collect the received video packets.
foreach (var videoSenderPacketData in videoPacketDataList)
{
if (videoSenderPacketData.frameId <= lastVideoFrameId)
{
continue;
}
if (!videoPacketCollections.ContainsKey(videoSenderPacketData.frameId))
{
videoPacketCollections[videoSenderPacketData.frameId] = new VideoSenderPacket[videoSenderPacketData.packetCount];
// Assign the largest new frame_id.
if (!addedFrameId.HasValue || addedFrameId < videoSenderPacketData.frameId)
{
addedFrameId = videoSenderPacketData.frameId;
}
}
videoPacketCollections[videoSenderPacketData.frameId][videoSenderPacketData.packetIndex] = videoSenderPacketData;
}
// Collect the received parity packets.
foreach (var paritySenderPacketData in parityPacketDataList)
{
if (paritySenderPacketData.frameId <= lastVideoFrameId)
{
continue;
}
if (!parityPacketCollections.ContainsKey(paritySenderPacketData.frameId))
{
int parityPacketCount = (paritySenderPacketData.videoPacketCount - 1) / FEC_GROUP_SIZE + 1;
parityPacketCollections[paritySenderPacketData.frameId] = new ParitySenderPacket[parityPacketCount];
}
parityPacketCollections[paritySenderPacketData.frameId][paritySenderPacketData.packetIndex] = paritySenderPacketData;
}
if (addedFrameId.HasValue)
{
foreach (var videoPacketCollection in videoPacketCollections)
{
int frameId = videoPacketCollection.Key;
VideoSenderPacket[] videoPackets = videoPacketCollection.Value;
// Skip the frame that just got added or even newer.
if (frameId >= addedFrameId)
{
continue;
}
// Find the parity packet collection corresponding to the video packet collection.
// Skip if there is no parity packet collection for the video frame.
if (!parityPacketCollections.ContainsKey(frameId))
{
continue;
}
ParitySenderPacket[] parityPackets = parityPacketCollections[frameId];
// Loop per each parity packet.
// Collect video packet indices to request.
List <int> videoPacketIndiecsToRequest = new List <int>();
List <int> parityPacketIndiecsToRequest = new List <int>();
for (int parityPacketIndex = 0; parityPacketIndex < parityPackets.Length; ++parityPacketIndex)
{
// Range of the video packets that correspond to the parity packet.
int videoPacketStartIndex = parityPacketIndex * FEC_GROUP_SIZE;
// Pick the end index with the end of video packet indices in mind (i.e., prevent overflow).
int videoPacketEndIndex = Math.Min(videoPacketStartIndex + FEC_GROUP_SIZE, videoPackets.Length);
// If the parity packet is missing, request all missing video packets and skip the FEC process.
// Also request the parity packet if there is a relevant missing video packet.
if (parityPackets[parityPacketIndex] == null)
{
bool parityPacketNeeded = false;
for (int videoPacketIndex = videoPacketStartIndex; videoPacketIndex < videoPacketEndIndex; ++videoPacketIndex)
{
if (videoPackets[videoPacketIndex] == null)
{
videoPacketIndiecsToRequest.Add(videoPacketIndex);
parityPacketNeeded = true;
}
}
if (parityPacketNeeded)
{
parityPacketIndiecsToRequest.Add(parityPacketIndex);
}
continue;
}
// Find if there is existing video packets and missing video packet indices.
// Check all video packets that relates to this parity packet.
var existingVideoPackets = new List <VideoSenderPacket>();
var missingVideoPacketIndices = new List <int>();
for (int videoPacketIndex = videoPacketStartIndex; videoPacketIndex < videoPacketEndIndex; ++videoPacketIndex)
{
if (videoPackets[videoPacketIndex] != null)
{
existingVideoPackets.Add(videoPackets[videoPacketIndex]);
}
else
{
missingVideoPacketIndices.Add(videoPacketIndex);
}
}
// Skip if there all video packets already exist.
if (missingVideoPacketIndices.Count == 0)
{
continue;
}
// XOR based FEC only works for a single missing packet.
if (missingVideoPacketIndices.Count > 1)
{
foreach (int missingIndex in missingVideoPacketIndices)
{
// Add the missing video packet indices for the vector to request them.
videoPacketIndiecsToRequest.Add(missingIndex);
}
continue;
}
// The missing video packet index.
int missingVideoPacketIndex = missingVideoPacketIndices[0];
// Reconstruct the missing video packet.
VideoSenderPacket fecVideoPacketData = new VideoSenderPacket();
fecVideoPacketData.frameId = frameId;
fecVideoPacketData.packetIndex = missingVideoPacketIndex;
fecVideoPacketData.packetCount = videoPackets.Length;
// Assign from the parity packet here since other video packets will be XOR'ed in the below loop.
fecVideoPacketData.messageData = parityPackets[parityPacketIndex].bytes;
foreach (var existingVideoPacket in existingVideoPackets)
{
for (int i = 0; i < fecVideoPacketData.messageData.Length; ++i)
{
fecVideoPacketData.messageData[i] ^= existingVideoPacket.messageData[i];
}
}
// Insert the reconstructed packet.
videoPacketCollections[frameId][missingVideoPacketIndex] = fecVideoPacketData;
}
// Request the video packets that FEC was not enough to fix.
udpSocket.Send(PacketUtils.createRequestReceiverPacketBytes(sessionId, frameId, false, videoPacketIndiecsToRequest, parityPacketIndiecsToRequest).bytes, remoteEndPoint);
}
}
// Find all full collections and their frame_ids.
var fullFrameIds = new List <int>();
foreach (var collectionPair in videoPacketCollections)
{
bool full = true;
foreach (var packetData in collectionPair.Value)
{
if (packetData == null)
{
full = false;
break;
}
}
if (full)
{
int frameId = collectionPair.Key;
fullFrameIds.Add(frameId);
}
}
// Extract messages from the full collections.
foreach (int fullFrameId in fullFrameIds)
{
var ms = new MemoryStream();
foreach (var packetData in videoPacketCollections[fullFrameId])
{
ms.Write(packetData.messageData, 0, packetData.messageData.Length);
}
var videoMessageData = VideoSenderMessage.Create(ms.ToArray());
videoMessages.Add(fullFrameId, videoMessageData);
videoPacketCollections.Remove(fullFrameId);
}
// Clean up frame_packet_collections.
var obsoleteFrameIds = new List <int>();
foreach (var collectionPair in videoPacketCollections)
{
if (collectionPair.Key <= lastVideoFrameId)
{
obsoleteFrameIds.Add(collectionPair.Key);
}
}
foreach (int obsoleteFrameId in obsoleteFrameIds)
{
videoPacketCollections.Remove(obsoleteFrameId);
}
}
