public void Init()
{
Clear();
opticalFlowCamerasController.GetDownsampleSize(out downsampleWidth, out downsampleHeight);
downsamplePixelBlend = opticalFlowCamerasController.downsamplePixelBlend;
downsamplePixelOpticalFlow = opticalFlowCamerasController.downsamplePixelOpticalFlow;
regionalSaliency = opticalFlowCamerasController.downsampleRegionalSaliency;
InitBuffer();//It only needs to be initialized once
InitPath();
isReady = true;
StartNewThread();
}
public void Init()
{
Clear();
opticalFlowCamerasController.GetDownsampleSize(out downsampleWidth, out downsampleHeight);
downsamplePixelBlend = opticalFlowCamerasController.downsamplePixelBlend;
downsamplePixelOpticalFlow = opticalFlowCamerasController.downsamplePixelOpticalFlow;
smoothPath = opticalFlowCamerasController.smoothPath;
int camNum = panelVideoController.cameraGroupNum;
pipList = new GameObject[camNum];
var canvasRt = canvas.GetComponent <RectTransform>();
for (int camId = 0; camId < camNum; camId++)
{
var tmp = Instantiate(pipPrefab);
var pipController = tmp.GetComponent <PipController>();
tmp.name = "pip" + camId;
pipController.id = camId;
var rt = tmp.GetComponent <RectTransform>();
//Debug.Log("init scale: " + rt.localScale);
tmp.transform.SetParent(canvas.transform);
//Debug.Log("init scale after set parent: " + rt.localScale);
rt.localPosition = Vector3.zero;
//Debug.Log("init scale after set localPosition: " + rt.localScale);
rt.localEulerAngles = Vector3.zero;
//Debug.Log("init scale after set localEulerAngles: " + rt.localScale);
rt.SetAsFirstSibling();
//Debug.Log("init scale after SetAsFirstSibling: " + rt.localScale);
rt.localScale = new Vector3(1, 1, 1);
//Debug.Log("init scale after set localScale: " + rt.localScale);
var ri = tmp.GetComponent <RawImage>();
if (panelVideoController.cameraNFOVs[camId].targetTexture != null)
{
var tmpTexture = panelVideoController.cameraNFOVs[camId].targetTexture;
var width = manager.cameraCalculate.targetTexture.width;
var height = (int)Mathf.Round(width * pipHeightWeight / pipWidthWeight);
panelVideoController.cameraNFOVs[camId].targetTexture = new RenderTexture(width, height, 24, RenderTextureFormat.ARGB32);
Debug.Log(string.Format("pip width:{0}, height:{1}", width, height));
Destroy(tmpTexture);
}
ri.texture = panelVideoController.cameraNFOVs[camId].targetTexture;
pipList[camId] = tmp;
Debug.Log("canvasRt.sizeDelta: " + canvasRt.sizeDelta);
var anchoredPos = new Vector2(canvasRt.sizeDelta.x, canvasRt.sizeDelta.y);
rt.anchoredPosition = anchoredPos;
Debug.Log("rt.positon: " + rt.position);
}
ResizePipsSize();
isReady = true;
}
public void PrepareIfCloseCameraWindow()
{
var path = opticalFlowCamerasController.smoothPath;
if (path == null)
{
Debug.LogError("No smooth path");
return;
}
InitDownsamplePixelBlendSaliencyMaskAndOpticalFlow();
var camNum = panelVideoController.cameraGroupNum;
var rectOfPixel = opticalFlowCamerasController.rectOfPixel;
int downsampleWidth, downsampleHeight;
opticalFlowCamerasController.GetDownsampleSize(out downsampleWidth, out downsampleHeight);
var regionalSaliency = opticalFlowCamerasController.regionalSaliency;
ifCloseCameraWindow = new List <bool> [camNum];
var totalKeyFrame = path[0].Count;
for (int i = 0; i < camNum; i++)
{
ifCloseCameraWindow[i] = new List <bool>();
for (int frame = 0; frame < totalKeyFrame; frame++)
{
ifCloseCameraWindow[i].Add(false);
}
}
var frameBeforeInitialIdleKeyFrame = new int[camNum];
for (int camId = 0; camId < camNum; camId++)
{
frameBeforeInitialIdleKeyFrame[camId] = -1;
}
var camValueList = new float[camNum];
for (int frame = 0; frame < totalKeyFrame; frame++)
{
for (int i = 0; i < camNum; i++)
{
var posTmp = path[i][frame];
camValueList[i] += downsamplePixelBlend[frame][(int)posTmp.x, (int)posTmp.y];
if (ifCloseCameraWindow[i][frame])
{
continue;
}
for (int j = i + 1; j < camNum; j++)
{
if (ifCloseCameraWindow[j][frame])
{
continue;
}
int overlapBeginFrame = frame;
int overlapEndFrame = frame - 1;
float sumI = 0;
float sumJ = 0;
for (int k = frame; k < totalKeyFrame; k++)
{
var posI = path[i][k];
var rectI = rectOfPixel[(int)posI.x, (int)posI.y];
var singleAreaI = OpticalFlowCamerasController.GetRectWidth(rectI) * OpticalFlowCamerasController.GetRectHeight(rectI);
var posJ = path[j][k];
var rectJ = rectOfPixel[(int)posJ.x, (int)posJ.y];
var singleAreaJ = OpticalFlowCamerasController.GetRectWidth(rectJ) * OpticalFlowCamerasController.GetRectHeight(rectJ);
var overlapArea = opticalFlowCamerasController.GetRectRectOverlapArea(rectI, rectJ);
var iou = overlapArea / (singleAreaI + singleAreaJ - overlapArea);
if (iou >= closeCameraWindowIouThreshold)
{
sumI += downsamplePixelBlend[k][(int)posI.x, (int)posI.y];
sumJ += downsamplePixelBlend[k][(int)posJ.x, (int)posJ.y];
overlapEndFrame = k;
}
else
{
break;
}
}
int closeCamId = sumI < sumJ ? i : j;
for (int k = overlapBeginFrame; k <= overlapEndFrame; k++)
{
ifCloseCameraWindow[closeCamId][k] = true;
}
}
}
}
float tmpMax = -1;
for (int i = 0; i < camValueList.Length; i++)
{
if (camValueList[i] > tmpMax)
{
tmpMax = camValueList[i];
mostValuableCamId = i;
}
}
for (int frame = 0; frame < totalKeyFrame; frame++)
{
for (int i = 0; i < camNum; i++)
{
if (ifCloseCameraWindow[i][frame])
{
continue;
}
var posI = path[i][frame];
bool ifUpdateCloseCameraWindow = false;
int frameAfterLastIdelKeyFrame = frame;
//Debug.Log("posI: " + posI);
if (downsamplePixelBlend[frame][(int)posI.x, (int)posI.y] >= minimalMeaningfulPixelSaliency || regionalSaliency[frame][(int)posI.x, (int)posI.y] >= minimalMeaningfulRegionalSaliency)
{
ifUpdateCloseCameraWindow = true;
}
else if (frame == totalKeyFrame - 1)
{
ifUpdateCloseCameraWindow = true;
frameAfterLastIdelKeyFrame = totalKeyFrame;
}
if (ifUpdateCloseCameraWindow)
{
float idleTime = ((float)frameAfterLastIdelKeyFrame - frameBeforeInitialIdleKeyFrame[i] - 1) / totalKeyFrame * videoPlayer.frameCount / videoPlayer.frameRate;
if (idleTime > allowedMaxIdleTime)
{
for (int j = frameBeforeInitialIdleKeyFrame[i] + 1; j < frameAfterLastIdelKeyFrame; j++)
{
ifCloseCameraWindow[i][j] = true;
}
}
frameBeforeInitialIdleKeyFrame[i] = frameAfterLastIdelKeyFrame;
}
}
}
for (int frame = 0; frame < totalKeyFrame; frame++)
{
int closedCamCnt = 0;
int chosenCamId = 0;
float maxV = 0;
for (int camId = 0; camId < camNum; camId++)
{
if (ifCloseCameraWindow[camId][frame])
{
closedCamCnt++;
}
var pos = path[camId][frame];
var nowV = Mathf.Max(downsamplePixelBlend[frame][(int)pos.x, (int)pos.y], regionalSaliency[frame][(int)pos.x, (int)pos.y]);
if (maxV < nowV)
{
maxV = nowV;
chosenCamId = camId;
}
}
if (closedCamCnt == camNum)
{
//Debug.Log(string.Format("ifCloseCameraWindow.count: {0}, frame: {1}, totalKeyFrame: {2}", ifCloseCameraWindow.Length, frame, totalKeyFrame));
//Debug.Log("ifCloseCameraWindow[chosenCamId].Count: " + ifCloseCameraWindow[chosenCamId].Count);
ifCloseCameraWindow[chosenCamId][frame] = false;
}
}
for (int camId = 0; camId < camNum; camId++)
{
//Debug.Log("Camera" + camId + ":");
for (int frame = 0; frame < totalKeyFrame; frame++)
{
var pos = path[camId][frame];
//Debug.Log(string.Format("blendPixelSaliency: {0}, regionalSaliency: {1}", downsamplePixelBlendSaliencyMask[frame][(int)pos.x, (int)pos.y], regionalSaliency[frame][(int)pos.x, (int)pos.y]));
}
//Debug.Log(".................");
}
Debug.Log("ifCloseCameraWindow[0].Count: " + ifCloseCameraWindow[0].Count);
Debug.Log("......................");
}
//Which camera windows of which frames are ready to close
public void PrepareIfCloseCameraWindow()
{
var path = opticalFlowCamerasController.smoothPath;
if (path == null)
{
Debug.Log("No smooth path");
return;
}
InitDownsamplePixelBlendSaliencyMaskAndOpticalFlow();
var camNum = panelVideoController.cameraGroupNum;
var rectOfPixel = opticalFlowCamerasController.rectOfPixel;
int downsampleWidth, downsampleHeight;
opticalFlowCamerasController.GetDownsampleSize(out downsampleWidth, out downsampleHeight);
var regionalSaliency = opticalFlowCamerasController.regionalSaliency;
ifCloseCameraWindow = new List <bool> [camNum];
var totalKeyFrame = path[0].Count;
for (int i = 0; i < camNum; i++)
{
ifCloseCameraWindow[i] = new List <bool>();
}
//Which is the previous frame of the idle frame
var frameBeforeInitialIdleKeyFrame = new int[camNum];
for (int camId = 0; camId < camNum; camId++)
{
frameBeforeInitialIdleKeyFrame[camId] = -1;
}
for (int frame = 0; frame < totalKeyFrame; frame++)
{
for (int i = 0; i < camNum; i++)
{
ifCloseCameraWindow[i].Add(false);
}
for (int i = 0; i < camNum; i++)
{
var posI = path[i][frame];
bool ifUpdateCloseCameraWindow = false;
int frameAfterLastIdelKeyFrame = frame;
if (downsamplePixelBlendSaliencyMask[frame][(int)posI.x, (int)posI.y] >= minimalMeaningfulPixelSaliency || regionalSaliency[frame][(int)posI.x, (int)posI.y] >= minimalMeaningfulRegionalSaliency)
{
ifUpdateCloseCameraWindow = true;
}
else if (frame == totalKeyFrame - 1)
{
ifUpdateCloseCameraWindow = true;
frameAfterLastIdelKeyFrame = totalKeyFrame;
}
if (ifUpdateCloseCameraWindow)
{
float idleTime = ((float)frameAfterLastIdelKeyFrame - frameBeforeInitialIdleKeyFrame[i] - 1) / totalKeyFrame * videoPlayer.frameCount / videoPlayer.frameRate;
if (idleTime > allowedMaxIdleTime)
{
for (int j = frameBeforeInitialIdleKeyFrame[i] + 1; j < frameAfterLastIdelKeyFrame; j++)
{
ifCloseCameraWindow[i][j] = true;
}
}
frameBeforeInitialIdleKeyFrame[i] = frameAfterLastIdelKeyFrame;
}
}
for (int i = 0; i < camNum; i++)
{
if (ifCloseCameraWindow[i][frame])
{
continue;
}
var posI = path[i][frame];
var rectI = rectOfPixel[(int)posI.x, (int)posI.y];
var singleAreaI = OpticalFlowCamerasController.GetRectWidth(rectI) * OpticalFlowCamerasController.GetRectHeight(rectI);
for (int j = i + 1; j < camNum; j++)
{
if (ifCloseCameraWindow[j][frame])
{
continue;
}
var posJ = path[j][frame];
var rectJ = rectOfPixel[(int)posJ.x, (int)posJ.y];
var singleAreaJ = OpticalFlowCamerasController.GetRectWidth(rectJ) * OpticalFlowCamerasController.GetRectHeight(rectJ);
var overlapArea = opticalFlowCamerasController.GetRectRectOverlapArea(rectI, rectJ);
var iou = overlapArea / (singleAreaI + singleAreaJ - overlapArea);
if (iou >= closeCameraWindowIouThreshold)
{
if (downsamplePixelBlendSaliencyMask[frame][(int)posI.x, (int)posI.y] < downsamplePixelBlendSaliencyMask[frame][(int)posJ.x, (int)posJ.y])
{
ifCloseCameraWindow[i][frame] = true;
}
else
{
ifCloseCameraWindow[j][frame] = true;
}
}
}
}
}
//Make sure that at least one camera is on for each frame
for (int frame = 0; frame < totalKeyFrame; frame++)
{
int closedCamCnt = 0;
int chosenCamId = 0;
float maxV = 0;
for (int camId = 0; camId < camNum; camId++)
{
if (ifCloseCameraWindow[camId][frame])
{
closedCamCnt++;
}
var pos = path[camId][frame];
var nowV = Mathf.Max(downsamplePixelBlendSaliencyMask[frame][(int)pos.x, (int)pos.y], regionalSaliency[frame][(int)pos.x, (int)pos.y]);
if (maxV < nowV)
{
maxV = nowV;
chosenCamId = camId;
}
}
if (closedCamCnt == camNum)//Restore the most meaningful camera
{
ifCloseCameraWindow[chosenCamId][frame] = false;
}
}
for (int camId = 0; camId < camNum; camId++)
{
Debug.Log("Camera" + camId + ":");
for (int frame = 0; frame < totalKeyFrame; frame++)
{
var pos = path[camId][frame];
Debug.Log(string.Format("blendPixelSaliency: {0}, regionalSaliency: {1}", downsamplePixelBlendSaliencyMask[frame][(int)pos.x, (int)pos.y], regionalSaliency[frame][(int)pos.x, (int)pos.y]));
}
Debug.Log(".................");
}
Debug.Log("ifCloseCameraWindow[0].Count: " + ifCloseCameraWindow[0].Count);
Debug.Log("......................");
}