//Interpolation on a rectangle
public Vector3 RectangleInterpolation(int currentKeyFrame, int currentFrame)
{
var totalKeyframe = downsamplePixelBlend.Count;
var l = currentKeyFrame - nowBeginKeyFrame;
var r = l + 1;
var chosenPath = nowSmoothPath;
var originSize = new Vector2(videoPlayer.targetTexture.width, videoPlayer.targetTexture.height);
int downsampleWidth, downsampleHeight;
opticalFlowCamerasController.GetDownsampleSize(out downsampleWidth, out downsampleHeight);
var al = chosenPath[l];
var tl = (float)(l + nowBeginKeyFrame) / totalKeyframe;
var w = Vector3.zero;
if (r == chosenPath.Count)//Only the last camera is left
{
w = panelVideoController.PixelToVector3(opticalFlowCamerasController.PixelToOriginPixelIntValue(chosenPath[l], new Vector2(downsampleWidth, downsampleHeight), originSize));
}
else
{
var ar = chosenPath[r];
var tr = (float)(r + nowBeginKeyFrame) / totalKeyframe;
var v = opticalFlowCamerasController.GetVector2Of2Pixels(al, ar, downsampleWidth);
var percentage = (float)currentFrame / videoPlayer.frameCount;
var tmp = (percentage - tl) / (tr - tl);
tmp = Mathf.Clamp(tmp, 0, 1);
v = v * tmp;
var p = al + v;
OpticalFlowCamerasController.NormalizePixelInRange(ref p.x, ref p.y, downsampleWidth, downsampleHeight);
var op = opticalFlowCamerasController.PixelToOriginPixelFloatValue(p, new Vector2(downsampleWidth, downsampleHeight), originSize);
w = panelVideoController.PixelToVector3(op);
}
return(w);
}
//FOV perception path planning
private IEnumerator FovAwarePathPlanning(List <Vector2> initialPath, int nextBeginKeyFrame, int nextStoredKeyFrameNum, List <Vector2> fovAwarePath)
{
if (initialPath != null && initialPath.Count > 0)
{
var frameNum = initialPath.Count;
var updateRadius = 10;//Radius used to update the path
var wp = 1e-4f;
fovAwarePath.Add(initialPath[0]);
int cntFrame = 0;
for (int frame = nextBeginKeyFrame + 1; frame < nextBeginKeyFrame + nextStoredKeyFrameNum; frame++)
{
var p = initialPath[frame - nextBeginKeyFrame];
var e = oo;
var pb = new Vector2();
for (int i = (int)p.x - updateRadius; i <= (int)p.x + updateRadius; i++)
{
for (int j = (int)p.y - updateRadius; j <= (int)p.y + updateRadius; j++)
{
if (j < 0 || j >= downsampleHeight)
{
continue;
}
var ii = i;
var jj = j;
OpticalFlowCamerasController.NormalizePixelInRange(ref ii, ref jj, downsampleWidth, downsampleHeight);
var pbTmp = new Vector2(ii, jj);
var v = opticalFlowCamerasController.GetVector2Of2Pixels(p, pbTmp, downsampleWidth);
float value = 0;
try
{
value = Mathf.Abs(1 - regionalSaliency[frame][ii, jj]) + wp * opticalFlowCamerasController.L1Norm(v);
}
catch {
Debug.LogError(string.Format("error! FovAwarePathPlanning regionalSaliency,regionalSaliency.count:{0},frame:{1}", regionalSaliency.Count, frame));
}
if (e > value)
{
e = value;
pb = pbTmp;
}
}
}
fovAwarePath.Add(pb);
cntFrame++;
if (cntFrame >= fovPathKeyFrameNumPerFrame)
{
cntFrame = 0;
yield return(null);
}
}
}
yield return(null);
}
public void UpdateMainNFOVCamera()
{
if (opticalFlowCamerasController.smoothPath == null || !videoPlayer.isPlaying)
{
return;
}
var originSize = new Vector2(videoPlayer.targetTexture.width, videoPlayer.targetTexture.height);
int downsampleWidth, downsampleHeight;
opticalFlowCamerasController.GetDownsampleSize(out downsampleWidth, out downsampleHeight);
var downsampleSize = new Vector2(downsampleWidth, downsampleHeight);
int nowFrame = (int)videoPlayer.frame;
var cameraNFOVs = panelVideoController.cameraNFOVs;
var nextFrame = KeyFrameToFrame(nextKeyFrame);
//Debug.Log(string.Format("oldFrame: {0}, nowFrame: {1}, nextFrame: {2}", oldFrame, nowFrame, nextFrame));
if (nextCameraId >= 0)
{
if (nowFrame < nextFrame || !reachable)
{
var path = opticalFlowCamerasController.smoothPath[nextCameraId];
var mForward = mainNFOVCamera.transform.forward;
var desP = path[nextKeyFrame];
var nowP = opticalFlowCamerasController.PixelToOriginPixelFloatValue(panelVideoController.EulerAngleToPixel(mainNFOVCamera.transform.eulerAngles), originSize, downsampleSize);
//Debug.Log(string.Format("frames: {0}, nowP: {1}, Time.deltaTime:{2}", (nowFrame - oldFrame), nowP, Time.deltaTime));
var v = GetVector2Of2Pixels(nowP, desP, downsampleWidth, selectedDirection);
var pixelDistPerFrame = nextFrame - oldFrame <= 0 ? 1e9f : v.magnitude / (nextFrame - oldFrame);
pixelDistPerFrame = Mathf.Min(moveMaxPixelPerFrame, pixelDistPerFrame);
var theta = Mathf.Max((nowFrame - oldFrame) * pixelDistPerFrame, 0);
var nv = v.normalized;
//var nextP = nowP + new Vector2(nv.x * theta, nv.y * theta);
var nextP = nowP + Mathf.Min(transferSpeed.magnitude, moveMaxPixelSpeed) * Time.deltaTime * transferSpeed.normalized;//只用帧数的话会比较卡顿
OpticalFlowCamerasController.NormalizePixelInRange(ref nextP.x, ref nextP.y, downsampleWidth, downsampleHeight);
var newForward = panelVideoController.PixelToVector3(opticalFlowCamerasController.PixelToOriginPixelFloatValue(nextP, downsampleSize, originSize));
mainNFOVCamera.transform.forward = newForward;
manager.panoramaVideoController.UpdateTextMethodStatus(2);
mainNFOVCamera.Render();
}
else
{
mainNFOVCamera.transform.forward = cameraNFOVs[nextCameraId].transform.forward;
manager.panoramaVideoController.UpdateTextMethodStatus(2);
mainNFOVCamera.Render();
nextKeyFrame = -1;
nowCameraId = nextCameraId;
selectedDirection = -1;
}
}
oldFrame = nowFrame;
}
//Update locator display
public void UpdateLocator()
{
if (locatorTexture == null)
{
return;
}
var videoWidth = videoPlayer.targetTexture.width;
var videoHeight = videoPlayer.targetTexture.height;
var width = (int)Mathf.Min(locatorTextureMaxLength, videoWidth * locatorTextureMaxLength / videoHeight);
var height = (int)Mathf.Min(locatorTextureMaxLength, videoHeight * locatorTextureMaxLength / videoWidth);
var size = new Vector2(width, height);
var originSize = new Vector2(videoWidth, videoHeight);
var angle = panelVideoController.EulerAngleToAngle(mainNFOVCamera.transform.eulerAngles);
var pixel = opticalFlowCamerasController.PixelToOriginPixelFloatValue(panelVideoController.AngleToPixel(angle), originSize, size);
int ox = (int)locatorPos.x;
int oy = (int)locatorPos.y;
int nx = (int)pixel.x;
int ny = (int)pixel.y;
foreach (var u in locatorShape)
{
int xx = (int)(ox + u.x);
int yy = (int)(oy + u.y);
OpticalFlowCamerasController.NormalizePixelInRange(ref xx, ref yy, width, height);
locatorTexture.SetPixel(xx, height - 1 - yy, Color.clear);
}
foreach (var u in locatorShape)
{
int xx = (int)(nx + u.x);
int yy = (int)(ny + u.y);
OpticalFlowCamerasController.NormalizePixelInRange(ref xx, ref yy, width, height);
locatorTexture.SetPixel(xx, height - 1 - yy, Color.red);
}
locatorTexture.Apply();
locatorPos = new Vector2(nx, ny);
}
//Limit the maximum moving speed to prevent the window from shaking too much
public Vector3 ApplyMaxSpeedLimit(Vector3 fa, Vector3 fb, float t)
{
if (fa.Equals(fb))
{
return(fa);
}
var originSize = new Vector2(videoPlayer.targetTexture.width, videoPlayer.targetTexture.height);
int downsampleWidth, downsampleHeight;
opticalFlowCamerasController.GetDownsampleSize(out downsampleWidth, out downsampleHeight);
var cameraCalculate = manager.cameraCalculate;
cameraCalculate.transform.forward = fa;
var ea = cameraCalculate.transform.eulerAngles;
var pa = panelVideoController.EulerAngleToPixel(ea);
pa = opticalFlowCamerasController.PixelToOriginPixelFloatValue(pa, originSize, new Vector2(downsampleWidth, downsampleHeight));
cameraCalculate.transform.forward = fb;
var eb = cameraCalculate.transform.eulerAngles;
var pb = panelVideoController.EulerAngleToPixel(eb);
pb = opticalFlowCamerasController.PixelToOriginPixelFloatValue(pb, originSize, new Vector2(downsampleWidth, downsampleHeight));
var v = opticalFlowCamerasController.GetVector2Of2Pixels(pa, pb, downsampleWidth);
if (v.magnitude == 0)
{
return(fa);
}
v = v / v.magnitude * Mathf.Min(maxSpeed * t, v.magnitude);
Debug.Log("ApplyMaxSpeedLimit");
Debug.Log(string.Format("t:{0}, v: {1}", t, v));
var p = pa + v;
OpticalFlowCamerasController.NormalizePixelInRange(ref p.x, ref p.y, downsampleWidth, downsampleHeight);
return(panelVideoController.PixelToVector3(opticalFlowCamerasController.PixelToOriginPixelFloatValue(p, new Vector2(downsampleWidth, downsampleHeight), originSize)));
}
