private void createSceneObjects()
{
kinectObject.transform.localScale = Vector3.one;
kinectObject.transform.localPosition = KinectPosition;
kinectObject.transform.eulerAngles = KinectRotation;
kinectObject.GetComponent <MeshRenderer>().material = new Material(Shader.Find("Diffuse"));
kinectObject.GetComponent <MeshRenderer>().material.color = SensorColor;
kinectObject.transform.name = "Kinect ID=" + ID;
debugTargetObject = GameObject.CreatePrimitive(PrimitiveType.Sphere);
debugTargetObject.transform.parent = kinectObject.transform;
setPlayerPosition_FromOrigin(InitPosition);
debugTargetObject.GetComponent <MeshRenderer>().material = new Material(Shader.Find("Diffuse"));
debugTargetObject.GetComponent <MeshRenderer>().material.color = SensorColor;
debugTargetObject.name = "debugTargetObject";
if (simulate)
{
debugTargetObject.AddComponent <TargetMoveSimulation>();
}
KinectReceiver receiver = kinectObject.AddComponent <KinectReceiver>();
receiver.startListening(Port);
if (!drawTarget)
{
debugTargetObject.GetComponent <MeshRenderer>().enabled = false;
kinectObject.GetComponent <MeshRenderer>().enabled = false;
}
}
void Awake()
{
InputState = InputState.IpAddress;
UiVisibility = true;
gestureRecognizer = new GestureRecognizer();
kinectReceiver = new KinectReceiver(azureKinectScreenMaterial, azureKinectScreen);
// Prepare a GestureRecognizer to recognize taps.
gestureRecognizer.Tapped += OnTapped;
gestureRecognizer.StartCapturingGestures();
statusText.text = "Waiting for user input.";
}
private async void TryConnectToKinectSender(IPEndPoint endPoint)
{
++connectingCount;
TextToaster.Toast($"Try connecting to a Sender at {endPoint}...");
var random = new System.Random();
int receiverSessionId;
while (true)
{
receiverSessionId = random.Next();
if (kinectReceivers.FirstOrDefault(x => x.ReceiverSessionId == receiverSessionId) == null)
{
break;
}
}
var kinectReceiver = new KinectReceiver(receiverSessionId, endPoint);
kinectReceivers.Add(kinectReceiver);
// Nudge the sender until a confirm packet is received.
for (int i = 0; i < 5; ++i)
{
if (kinectReceiver.State != PrepareState.Unprepared)
{
--connectingCount;
return;
}
udpSocket.Send(PacketHelper.createConnectReceiverPacketBytes(receiverSessionId, true, true), endPoint);
print($"Sent connect packet #{i}");
await Task.Delay(300);
}
// Give up and forget about the connection if a confirm packet has not been received after all the connect packets.
if (kinectReceiver.State == PrepareState.Unprepared)
{
kinectReceivers.Remove(kinectReceiver);
}
--connectingCount;
}
public void Run(KinectReceiver kinectReceiver,
int senderSessionId,
ConcurrentQueue <VideoSenderPacketData> videoPacketDataQueue,
ConcurrentQueue <FecSenderPacketData> fecPacketDataQueue,
ConcurrentQueue <AudioSenderPacketData> audioPacketDataQueue)
{
SocketError error = SocketError.WouldBlock;
while (true)
{
var packet = kinectReceiver.udpSocket.Receive(out error);
if (packet == null)
{
break;
}
int sessionId = PacketHelper.getSessionIdFromSenderPacketBytes(packet);
var packetType = PacketHelper.getPacketTypeFromSenderPacketBytes(packet);
if (sessionId != senderSessionId)
{
continue;
}
if (packetType == SenderPacketType.Frame)
{
videoPacketDataQueue.Enqueue(VideoSenderPacketData.Parse(packet));
}
else if (packetType == SenderPacketType.Fec)
{
fecPacketDataQueue.Enqueue(FecSenderPacketData.Parse(packet));
}
else if (packetType == SenderPacketType.Audio)
{
audioPacketDataQueue.Enqueue(AudioSenderPacketData.Parse(packet));
}
}
if (error != SocketError.WouldBlock)
{
UnityEngine.Debug.Log($"Error from receiving packets: {error.ToString()}");
}
}
public void Run(KinectReceiver kinectReceiver,
ConcurrentQueue <AudioSenderPacketData> audioPacketDataQueue)
{
var audioPacketDataSet = new List <AudioSenderPacketData>();
{
AudioSenderPacketData audioPacketData;
while (audioPacketDataQueue.TryDequeue(out audioPacketData))
{
audioPacketDataSet.Add(audioPacketData);
}
}
audioPacketDataSet.Sort((x, y) => x.frameId.CompareTo(y.frameId));
float[] pcm = new float[KH_SAMPLES_PER_FRAME * KH_CHANNEL_COUNT];
int index = 0;
while (kinectReceiver.ringBuffer.FreeSamples >= pcm.Length)
{
if (index >= audioPacketDataSet.Count)
{
break;
}
var audioPacketData = audioPacketDataSet[index++];
if (audioPacketData.frameId <= kinectReceiver.lastAudioFrameId)
{
continue;
}
kinectReceiver.audioDecoder.Decode(audioPacketData.opusFrame, pcm, KH_SAMPLES_PER_FRAME);
kinectReceiver.ringBuffer.Write(pcm);
kinectReceiver.lastAudioFrameId = audioPacketData.frameId;
}
}
public void Run(KinectReceiver kinectReceiver,
ConcurrentQueue <VideoSenderPacketData> videoPacketDataQueue,
ConcurrentQueue <FecSenderPacketData> fecPacketDataQueue)
{
// The logic for XOR FEC packets are almost the same to frame packets.
// The operations for XOR FEC packets should happen before the frame packets
// so that frame packet can be created with XOR FEC packets when a missing
// frame packet is detected.
FecSenderPacketData fecSenderPacketData;
while (fecPacketDataQueue.TryDequeue(out fecSenderPacketData))
{
if (fecSenderPacketData.frameId <= kinectReceiver.lastVideoFrameId)
{
continue;
}
if (!fecPacketCollections.ContainsKey(fecSenderPacketData.frameId))
{
fecPacketCollections[fecSenderPacketData.frameId] = new FecSenderPacketData[fecSenderPacketData.packetCount];
}
fecPacketCollections[fecSenderPacketData.frameId][fecSenderPacketData.packetIndex] = fecSenderPacketData;
}
VideoSenderPacketData videoSenderPacketData;
while (videoPacketDataQueue.TryDequeue(out videoSenderPacketData))
{
if (videoSenderPacketData.frameId <= kinectReceiver.lastVideoFrameId)
{
continue;
}
// If there is a packet for a new frame, check the previous frames, and if
// there is a frame with missing packets, try to create them using xor packets.
// If using the xor packets fails, request the sender to retransmit the packets.
if (!videoPacketCollections.ContainsKey(videoSenderPacketData.frameId))
{
videoPacketCollections[videoSenderPacketData.frameId] = new VideoSenderPacketData[videoSenderPacketData.packetCount];
///////////////////////////////////
// Forward Error Correction Start//
///////////////////////////////////
// Request missing packets of the previous frames.
foreach (var collectionPair in videoPacketCollections)
{
if (collectionPair.Key < videoSenderPacketData.frameId)
{
int missingFrameId = collectionPair.Key;
//var missingPacketIndices = collectionPair.Value.GetMissingPacketIds();
var missingPacketIndices = new List <int>();
for (int i = 0; i < collectionPair.Value.Length; ++i)
{
if (collectionPair.Value[i] == null)
{
missingPacketIndices.Add(i);
}
}
// Try correction using XOR FEC packets.
var fecFailedPacketIndices = new List <int>();
var fecPacketIndices = new List <int>();
// missing_packet_index cannot get error corrected if there is another missing_packet_index
// that belongs to the same XOR FEC packet...
foreach (int i in missingPacketIndices)
{
bool found = false;
foreach (int j in missingPacketIndices)
{
if (i == j)
{
continue;
}
if ((i / XOR_MAX_GROUP_SIZE) == (j / XOR_MAX_GROUP_SIZE))
{
found = true;
break;
}
}
if (found)
{
fecFailedPacketIndices.Add(i);
}
else
{
fecPacketIndices.Add(i);
}
}
foreach (int fecPacketIndex in fecPacketIndices)
{
// Try getting the XOR FEC packet for correction.
int xorPacketIndex = fecPacketIndex / XOR_MAX_GROUP_SIZE;
if (!fecPacketCollections.ContainsKey(missingFrameId))
{
fecFailedPacketIndices.Add(fecPacketIndex);
continue;
}
var fecPacketData = fecPacketCollections[missingFrameId][xorPacketIndex];
// Give up if there is no xor packet yet.
if (fecPacketData == null)
{
fecFailedPacketIndices.Add(fecPacketIndex);
continue;
}
var fecVideoPacketData = new VideoSenderPacketData();
fecVideoPacketData.frameId = missingFrameId;
fecVideoPacketData.packetIndex = videoSenderPacketData.packetIndex;
fecVideoPacketData.packetCount = videoSenderPacketData.packetCount;
fecVideoPacketData.messageData = fecPacketData.bytes;
int beginFramePacketIndex = xorPacketIndex * XOR_MAX_GROUP_SIZE;
int endFramePacketIndex = Math.Min(beginFramePacketIndex + XOR_MAX_GROUP_SIZE, collectionPair.Value.Length);
// Run bitwise XOR with all other packets belonging to the same XOR FEC packet.
for (int i = beginFramePacketIndex; i < endFramePacketIndex; ++i)
{
if (i == fecPacketIndex)
{
continue;
}
for (int j = 0; j < fecVideoPacketData.messageData.Length; ++j)
{
fecVideoPacketData.messageData[j] ^= collectionPair.Value[i].messageData[j];
}
}
videoPacketCollections[missingFrameId][fecPacketIndex] = fecVideoPacketData;
} // end of foreach (int missingPacketIndex in missingPacketIndices)
kinectReceiver.udpSocket.Send(PacketHelper.createRequestReceiverPacketBytes(collectionPair.Key, fecFailedPacketIndices));
}
}
/////////////////////////////////
// Forward Error Correction End//
/////////////////////////////////
}
// End of if (frame_packet_collections.find(frame_id) == frame_packet_collections.end())
// which was for reacting to a packet for a new frame.
videoPacketCollections[videoSenderPacketData.frameId][videoSenderPacketData.packetIndex] = videoSenderPacketData;
}
// Find all full collections and their frame_ids.
var fullFrameIds = new List <int>();
foreach (var collectionPair in videoPacketCollections)
{
bool full = true;
foreach (var packetData in collectionPair.Value)
{
if (packetData == null)
{
full = false;
break;
}
}
if (full)
{
int frameId = collectionPair.Key;
fullFrameIds.Add(frameId);
}
}
// Extract messages from the full collections.
foreach (int fullFrameId in fullFrameIds)
{
var ms = new MemoryStream();
foreach (var packetData in videoPacketCollections[fullFrameId])
{
ms.Write(packetData.messageData, 0, packetData.messageData.Length);
}
var videoMessageData = VideoSenderMessageData.Parse(ms.ToArray());
kinectReceiver.videoMessageQueue.Enqueue(new Tuple <int, VideoSenderMessageData>(fullFrameId, videoMessageData));
videoPacketCollections.Remove(fullFrameId);
}
// Clean up frame_packet_collections.
var obsoleteFrameIds = new List <int>();
foreach (var collectionPair in videoPacketCollections)
{
if (collectionPair.Key <= kinectReceiver.lastVideoFrameId)
{
obsoleteFrameIds.Add(collectionPair.Key);
}
}
foreach (int obsoleteFrameId in obsoleteFrameIds)
{
videoPacketCollections.Remove(obsoleteFrameId);
}
}
void Update()
{
// Sends virtual keyboards strokes to the TextMeshes for the IP address and the port.
AbsorbInput();
if (Input.GetKeyDown(KeyCode.Tab))
{
if (connectionWindow.ConnectionTarget == ConnectionTarget.Controller)
{
connectionWindow.ConnectionTarget = ConnectionTarget.Kinect;
}
else
{
connectionWindow.ConnectionTarget = ConnectionTarget.Controller;
}
}
if (Input.GetKeyDown(KeyCode.Return) || Input.GetKeyDown(KeyCode.KeypadEnter))
{
if (connectionWindow.ConnectionTarget == ConnectionTarget.Controller)
{
TryConnectToController();
}
else
{
StartCoroutine(TryConnectToKinect());
}
}
// Gives the information of the camera position and floor level.
if (Input.GetKeyDown(KeyCode.Space))
{
sharedSpaceAnchor.UpdateTransform(cameraTransform.position, cameraTransform.rotation);
}
if (Input.GetKeyDown(KeyCode.D))
{
sharedSpaceAnchor.DebugVisibility = !sharedSpaceAnchor.DebugVisibility;
}
if (controllerClient != null)
{
var receiverStates = new List <ReceiverState>();
if (kinectReceiver != null)
{
var receiverState = new ReceiverState(kinectReceiver.SenderEndPoint.Address.ToString(),
kinectReceiver.SenderEndPoint.Port,
kinectReceiver.ReceiverSessionId);
receiverStates.Add(receiverState);
}
try
{
controllerClient.SendViewerState(receiverStates);
}
catch (TcpSocketException e)
{
print($"TcpSocketException while connecting: {e}");
controllerClient = null;
}
}
if (kinectReceiver != null)
{
var senderPacketSet = SenderPacketReceiver.Receive(udpSocket);
if (!kinectReceiver.UpdateFrame(udpSocket, senderPacketSet))
{
kinectReceiver = null;
ConnectWindowVisibility = true;
}
}
}
// To copy the c++ receiver, for easier development,
// there should be only one chance to send a ping.
private IEnumerator TryConnectToKinect()
{
if (!ConnectWindowVisibility)
{
textToaster.Toast("Cannot try connecting to more than one remote machine.");
yield break;
}
ConnectWindowVisibility = false;
// The default IP address is 127.0.0.1.
string ipAddressText = connectionWindow.IpAddressInputText;
if (ipAddressText.Length == 0)
{
ipAddressText = "127.0.0.1";
}
string logString = $"Try connecting to {ipAddressText}...";
textToaster.Toast(logString);
statusText.text = logString;
var random = new System.Random();
int receiverSessionId = random.Next();
var ipAddress = IPAddress.Parse(ipAddressText);
var endPoint = new IPEndPoint(ipAddress, SENDER_PORT);
InitSenderPacketData initPacketData;
int connectCount = 0;
while (true)
{
udpSocket.Send(PacketHelper.createConnectReceiverPacketBytes(receiverSessionId, true, true, true), endPoint);
++connectCount;
print("Sent connect packet");
yield return(new WaitForSeconds(0.3f));
try
{
var senderPacketSet = SenderPacketReceiver.Receive(udpSocket);
if (senderPacketSet.InitPacketDataList.Count > 0)
{
initPacketData = senderPacketSet.InitPacketDataList[0];
break;
}
}
catch (UdpSocketException e)
{
print($"UdpSocketException while connecting: {e}");
}
if (connectCount == 10)
{
textToaster.Toast("Tried connected 10 times but failed to receive an init packet...\n");
ConnectWindowVisibility = true;
yield break;
}
}
textToaster.Toast("Start creating screen");
if (sharedSpaceAnchor.KinectOrigin == null)
{
sharedSpaceAnchor.AddKinectOrigin();
}
yield return(StartCoroutine(sharedSpaceAnchor.KinectOrigin.Screen.SetupMesh(initPacketData)));
sharedSpaceAnchor.KinectOrigin.Speaker.Setup();
kinectReceiver = new KinectReceiver(receiverSessionId, endPoint, sharedSpaceAnchor.KinectOrigin, initPacketData);
}
