Example #1
0
    //Interpolation on a rectangle
    public Vector3 RectangleInterpolation(int currentKeyFrame, int currentFrame)
    {
        var totalKeyframe = downsamplePixelBlend.Count;
        var l = currentKeyFrame - nowBeginKeyFrame;
        var r = l + 1;
        var chosenPath = nowSmoothPath;
        var originSize = new Vector2(videoPlayer.targetTexture.width, videoPlayer.targetTexture.height);
        int downsampleWidth, downsampleHeight;

        opticalFlowCamerasController.GetDownsampleSize(out downsampleWidth, out downsampleHeight);
        var al = chosenPath[l];
        var tl = (float)(l + nowBeginKeyFrame) / totalKeyframe;
        var w  = Vector3.zero;

        if (r == chosenPath.Count)//Only the last camera is left
        {
            w = panelVideoController.PixelToVector3(opticalFlowCamerasController.PixelToOriginPixelIntValue(chosenPath[l], new Vector2(downsampleWidth, downsampleHeight), originSize));
        }
        else
        {
            var ar         = chosenPath[r];
            var tr         = (float)(r + nowBeginKeyFrame) / totalKeyframe;
            var v          = opticalFlowCamerasController.GetVector2Of2Pixels(al, ar, downsampleWidth);
            var percentage = (float)currentFrame / videoPlayer.frameCount;
            var tmp        = (percentage - tl) / (tr - tl);
            tmp = Mathf.Clamp(tmp, 0, 1);
            v   = v * tmp;
            var p = al + v;
            OpticalFlowCamerasController.NormalizePixelInRange(ref p.x, ref p.y, downsampleWidth, downsampleHeight);
            var op = opticalFlowCamerasController.PixelToOriginPixelFloatValue(p, new Vector2(downsampleWidth, downsampleHeight), originSize);
            w = panelVideoController.PixelToVector3(op);
        }
        return(w);
    }
    private void Awake()
    {
        manager                      = GameObject.Find("Manager").GetComponent <Manager>();
        mainNFOV23Content            = manager.mainNFOV23Content;
        cameraCalculate              = manager.cameraCalculate;
        panelVideoController         = manager.panelVideoController;
        opticalFlowCamerasController = manager.opticalFlowCamerasController;
        videoPlayer                  = manager.videoPlayer;

        nowCameraId       = -1;
        nextCameraId      = -1;
        reachable         = true;
        oldFrame          = 0;
        nextKeyFrame      = -1;
        selectedDirection = -1;


        arrowShowTime = 0;

        closeCameraWindowIouThreshold     = 0.5f;
        allowedMaxIdleTime                = 3;
        minimalMeaningfulPixelSaliency    = 0.001f;
        minimalMeaningfulRegionalSaliency = 0.001f;
        maxTransferKeyFrameNum            = 50;
        moveMaxPixelSpeed    = 30;
        moveMaxPixelPerFrame = moveMaxPixelSpeed / videoPlayer.frameRate;
        Debug.Log("SceneNumber: " + Manager.GetActivateSceneNumber());
        transferSpeed = Vector2.zero;
    }
Example #3
0
    void Awake()
    {
#if UNITY_EDITOR
#else
        if (Manager.GetActivateSceneNumber() == 2)
        {
            Screen.SetResolution(display1Width, display1Height, false);
        }
        else if (Manager.GetActivateSceneNumber() == 3)
        {
            Screen.SetResolution(display2Width, display2Height, false);
        }
#endif
        manager                      = GameObject.Find("Manager").GetComponent <Manager>();
        videoPlayer                  = manager.videoPlayer;
        menubarController            = manager.menuBar.GetComponent <MenuBarController>();
        panelVideoController         = manager.panelVideoController;
        opticalFlowCamerasController = manager.opticalFlowCamerasController;

        var dontDestroyOnLoadGameObj = GameObject.Find("DontDestroyOnLoad");
        if (dontDestroyOnLoadGameObj != null)
        {
            dontDestroyOnLoad = dontDestroyOnLoadGameObj.GetComponent <MyDontDestroyOnLoad>();
        }

        var sceneNumber = Manager.GetActivateSceneNumber();
        if (sceneNumber >= 2)
        {
            PrepareCovers();
        }

        UpdateTextMethodStatus(0);
        if (dontDestroyOnLoadGameObj == null)
        {
            dontDestroyOnLoadGameObj = new GameObject("DontDestroyOnLoad");
            dontDestroyOnLoad        = dontDestroyOnLoadGameObj.AddComponent <MyDontDestroyOnLoad>();
            //Get the window handle.
            windowPtr = FindWindow(null, Manager.productName);
            dontDestroyOnLoad.windowPtr = windowPtr;
            DontDestroyOnLoad(dontDestroyOnLoad);
            if (panelTestStartCover != null)
            {
                panelTestStartCover.SetActive(true);
            }
        }
        else
        {
            windowPtr = dontDestroyOnLoad.windowPtr;
            if (panelTestStartCover != null)
            {
                panelTestStartCover.SetActive(false);
            }
            dontDestroyOnLoad.RestoreData(manager);
        }
        manager.textMethodStatus.gameObject.SetActive(true);
        minDragDist = 1;
    }
Example #4
0
    public bool isReady;//Is initialization called

    private void Awake()
    {
        manager = GameObject.Find("Manager").GetComponent <Manager>();
        opticalFlowCamerasController = manager.opticalFlowCamerasController;
        panelVideoController         = manager.panelVideoController;
        videoPlayer = manager.videoPlayer;
        initialPathKeyFrameNumPerFrame = 10;
        fovPathKeyFrameNumPerFrame     = 10;
        maxSpeed = 30;
        isReady  = false;
    }
Example #5
0
 //FOV perception path planning
 private IEnumerator FovAwarePathPlanning(List <Vector2> initialPath, int nextBeginKeyFrame, int nextStoredKeyFrameNum, List <Vector2> fovAwarePath)
 {
     if (initialPath != null && initialPath.Count > 0)
     {
         var frameNum     = initialPath.Count;
         var updateRadius = 10;//Radius used to update the path
         var wp           = 1e-4f;
         fovAwarePath.Add(initialPath[0]);
         int cntFrame = 0;
         for (int frame = nextBeginKeyFrame + 1; frame < nextBeginKeyFrame + nextStoredKeyFrameNum; frame++)
         {
             var p  = initialPath[frame - nextBeginKeyFrame];
             var e  = oo;
             var pb = new Vector2();
             for (int i = (int)p.x - updateRadius; i <= (int)p.x + updateRadius; i++)
             {
                 for (int j = (int)p.y - updateRadius; j <= (int)p.y + updateRadius; j++)
                 {
                     if (j < 0 || j >= downsampleHeight)
                     {
                         continue;
                     }
                     var ii = i;
                     var jj = j;
                     OpticalFlowCamerasController.NormalizePixelInRange(ref ii, ref jj, downsampleWidth, downsampleHeight);
                     var   pbTmp = new Vector2(ii, jj);
                     var   v     = opticalFlowCamerasController.GetVector2Of2Pixels(p, pbTmp, downsampleWidth);
                     float value = 0;
                     try
                     {
                         value = Mathf.Abs(1 - regionalSaliency[frame][ii, jj]) + wp * opticalFlowCamerasController.L1Norm(v);
                     }
                     catch {
                         Debug.LogError(string.Format("error! FovAwarePathPlanning regionalSaliency,regionalSaliency.count:{0},frame:{1}", regionalSaliency.Count, frame));
                     }
                     if (e > value)
                     {
                         e  = value;
                         pb = pbTmp;
                     }
                 }
             }
             fovAwarePath.Add(pb);
             cntFrame++;
             if (cntFrame >= fovPathKeyFrameNumPerFrame)
             {
                 cntFrame = 0;
                 yield return(null);
             }
         }
     }
     yield return(null);
 }
    public void UpdateMainNFOVCamera()
    {
        if (opticalFlowCamerasController.smoothPath == null || !videoPlayer.isPlaying)
        {
            return;
        }
        var originSize = new Vector2(videoPlayer.targetTexture.width, videoPlayer.targetTexture.height);
        int downsampleWidth, downsampleHeight;

        opticalFlowCamerasController.GetDownsampleSize(out downsampleWidth, out downsampleHeight);
        var downsampleSize = new Vector2(downsampleWidth, downsampleHeight);
        int nowFrame       = (int)videoPlayer.frame;
        var cameraNFOVs    = panelVideoController.cameraNFOVs;
        var nextFrame      = KeyFrameToFrame(nextKeyFrame);

        //Debug.Log(string.Format("oldFrame: {0}, nowFrame: {1}, nextFrame: {2}", oldFrame, nowFrame, nextFrame));
        if (nextCameraId >= 0)
        {
            if (nowFrame < nextFrame || !reachable)
            {
                var path     = opticalFlowCamerasController.smoothPath[nextCameraId];
                var mForward = mainNFOVCamera.transform.forward;
                var desP     = path[nextKeyFrame];
                var nowP     = opticalFlowCamerasController.PixelToOriginPixelFloatValue(panelVideoController.EulerAngleToPixel(mainNFOVCamera.transform.eulerAngles), originSize, downsampleSize);
                //Debug.Log(string.Format("frames: {0}, nowP: {1}, Time.deltaTime:{2}", (nowFrame - oldFrame), nowP, Time.deltaTime));
                var v = GetVector2Of2Pixels(nowP, desP, downsampleWidth, selectedDirection);
                var pixelDistPerFrame = nextFrame - oldFrame <= 0 ? 1e9f : v.magnitude / (nextFrame - oldFrame);
                pixelDistPerFrame = Mathf.Min(moveMaxPixelPerFrame, pixelDistPerFrame);
                var theta = Mathf.Max((nowFrame - oldFrame) * pixelDistPerFrame, 0);
                var nv    = v.normalized;
                //var nextP = nowP + new Vector2(nv.x * theta, nv.y * theta);
                var nextP = nowP + Mathf.Min(transferSpeed.magnitude, moveMaxPixelSpeed) * Time.deltaTime * transferSpeed.normalized;//只用帧数的话会比较卡顿
                OpticalFlowCamerasController.NormalizePixelInRange(ref nextP.x, ref nextP.y, downsampleWidth, downsampleHeight);
                var newForward = panelVideoController.PixelToVector3(opticalFlowCamerasController.PixelToOriginPixelFloatValue(nextP, downsampleSize, originSize));
                mainNFOVCamera.transform.forward = newForward;
                manager.panoramaVideoController.UpdateTextMethodStatus(2);
                mainNFOVCamera.Render();
            }
            else
            {
                mainNFOVCamera.transform.forward = cameraNFOVs[nextCameraId].transform.forward;
                manager.panoramaVideoController.UpdateTextMethodStatus(2);
                mainNFOVCamera.Render();
                nextKeyFrame      = -1;
                nowCameraId       = nextCameraId;
                selectedDirection = -1;
            }
        }
        oldFrame = nowFrame;
    }
    private void Awake()
    {
        Debug.Log("MainNFOVController.Awake");
        manager                      = GameObject.Find("Manager").GetComponent <Manager>();
        mainNFOVContent              = manager.mainNFOVContent;
        cameraCalculate              = manager.cameraCalculate;
        panelVideoController         = manager.panelVideoController;
        opticalFlowCamerasController = manager.opticalFlowCamerasController;
        videoPlayer                  = manager.videoPlayer;

        nowCameraId         = 0;
        nextCameraId        = 0;
        mainNFOVCamera      = Instantiate(cameraCalculate);
        mainNFOVCamera.name = "MainNFOVCamera";
        var t = Instantiate(cameraCalculate.targetTexture);

        t.name = "MainNFOVCameraTexture";
        mainNFOVCamera.targetTexture = t;
        var ri = mainNFOVContent.GetComponent <RawImage>();

        ri.texture   = t;
        reachable    = true;
        oldFrame     = 0;
        nextKeyFrame = -1;
        locatorPos   = Vector2.zero;
        locatorShape = new List <Vector2>();
        for (int i = -5; i <= 5; i++)
        {
            for (int j = -1; j <= 1; j++)
            {
                locatorShape.Add(new Vector2(i, j));
                locatorShape.Add(new Vector2(j, i));
            }
        }
        selectedDirection = -1;

        arrowShowTime = 0;

        closeCameraWindowIouThreshold     = 0.3f;
        allowedMaxIdleTime                = 3;
        minimalMeaningfulPixelSaliency    = 0.02f;
        minimalMeaningfulRegionalSaliency = 0.02f;
        maxTransferKeyFrameNum            = 25;
        Debug.Log("SceneNumber: " + Manager.GetActivateSceneNumber());
    }
Example #8
0
 private void Awake()
 {
     manager = GameObject.Find("Manager").GetComponent <Manager>();
     opticalFlowCamerasController = manager.opticalFlowCamerasController;
     panelVideoController         = manager.panelVideoController;
     videoPlayer        = manager.videoPlayer;
     mainCamera         = manager.mainCamera;
     isReady            = false;
     pipSizeMultiple    = 0.3f;
     pipWidthWeight     = 1;
     pipHeightWeight    = 1;
     minHorizentalAngle = 40;
     minVerticalAngle   = 40;
     maxTilt            = 120;
     maxDepth           = 50;
     minDepth           = -20;
     canPauseUpdate     = true;
     var rt         = canvas.GetComponent <RectTransform>();
     var canvasSize = rt.sizeDelta;
 }
    //Update locator display
    public void UpdateLocator()
    {
        if (locatorTexture == null)
        {
            return;
        }
        var videoWidth  = videoPlayer.targetTexture.width;
        var videoHeight = videoPlayer.targetTexture.height;
        var width       = (int)Mathf.Min(locatorTextureMaxLength, videoWidth * locatorTextureMaxLength / videoHeight);
        var height      = (int)Mathf.Min(locatorTextureMaxLength, videoHeight * locatorTextureMaxLength / videoWidth);
        var size        = new Vector2(width, height);
        var originSize  = new Vector2(videoWidth, videoHeight);
        var angle       = panelVideoController.EulerAngleToAngle(mainNFOVCamera.transform.eulerAngles);
        var pixel       = opticalFlowCamerasController.PixelToOriginPixelFloatValue(panelVideoController.AngleToPixel(angle), originSize, size);
        int ox          = (int)locatorPos.x;
        int oy          = (int)locatorPos.y;
        int nx          = (int)pixel.x;
        int ny          = (int)pixel.y;

        foreach (var u in locatorShape)
        {
            int xx = (int)(ox + u.x);
            int yy = (int)(oy + u.y);
            OpticalFlowCamerasController.NormalizePixelInRange(ref xx, ref yy, width, height);
            locatorTexture.SetPixel(xx, height - 1 - yy, Color.clear);
        }
        foreach (var u in locatorShape)
        {
            int xx = (int)(nx + u.x);
            int yy = (int)(ny + u.y);
            OpticalFlowCamerasController.NormalizePixelInRange(ref xx, ref yy, width, height);
            locatorTexture.SetPixel(xx, height - 1 - yy, Color.red);
        }
        locatorTexture.Apply();
        locatorPos = new Vector2(nx, ny);
    }
Example #10
0
    //Limit the maximum moving speed to prevent the window from shaking too much
    public Vector3 ApplyMaxSpeedLimit(Vector3 fa, Vector3 fb, float t)
    {
        if (fa.Equals(fb))
        {
            return(fa);
        }
        var originSize = new Vector2(videoPlayer.targetTexture.width, videoPlayer.targetTexture.height);
        int downsampleWidth, downsampleHeight;

        opticalFlowCamerasController.GetDownsampleSize(out downsampleWidth, out downsampleHeight);
        var cameraCalculate = manager.cameraCalculate;

        cameraCalculate.transform.forward = fa;
        var ea = cameraCalculate.transform.eulerAngles;
        var pa = panelVideoController.EulerAngleToPixel(ea);

        pa = opticalFlowCamerasController.PixelToOriginPixelFloatValue(pa, originSize, new Vector2(downsampleWidth, downsampleHeight));
        cameraCalculate.transform.forward = fb;
        var eb = cameraCalculate.transform.eulerAngles;
        var pb = panelVideoController.EulerAngleToPixel(eb);

        pb = opticalFlowCamerasController.PixelToOriginPixelFloatValue(pb, originSize, new Vector2(downsampleWidth, downsampleHeight));
        var v = opticalFlowCamerasController.GetVector2Of2Pixels(pa, pb, downsampleWidth);

        if (v.magnitude == 0)
        {
            return(fa);
        }
        v = v / v.magnitude * Mathf.Min(maxSpeed * t, v.magnitude);
        Debug.Log("ApplyMaxSpeedLimit");
        Debug.Log(string.Format("t:{0}, v: {1}", t, v));
        var p = pa + v;

        OpticalFlowCamerasController.NormalizePixelInRange(ref p.x, ref p.y, downsampleWidth, downsampleHeight);
        return(panelVideoController.PixelToVector3(opticalFlowCamerasController.PixelToOriginPixelFloatValue(p, new Vector2(downsampleWidth, downsampleHeight), originSize)));
    }
    public void PrepareIfCloseCameraWindow()
    {
        var path = opticalFlowCamerasController.smoothPath;

        if (path == null)
        {
            Debug.LogError("No smooth path");
            return;
        }
        InitDownsamplePixelBlendSaliencyMaskAndOpticalFlow();
        var camNum = panelVideoController.cameraGroupNum;
        var rectOfPixel = opticalFlowCamerasController.rectOfPixel;
        int downsampleWidth, downsampleHeight;

        opticalFlowCamerasController.GetDownsampleSize(out downsampleWidth, out downsampleHeight);
        var regionalSaliency = opticalFlowCamerasController.regionalSaliency;

        ifCloseCameraWindow = new List <bool> [camNum];
        var totalKeyFrame = path[0].Count;

        for (int i = 0; i < camNum; i++)
        {
            ifCloseCameraWindow[i] = new List <bool>();
            for (int frame = 0; frame < totalKeyFrame; frame++)
            {
                ifCloseCameraWindow[i].Add(false);
            }
        }

        var frameBeforeInitialIdleKeyFrame = new int[camNum];

        for (int camId = 0; camId < camNum; camId++)
        {
            frameBeforeInitialIdleKeyFrame[camId] = -1;
        }

        var camValueList = new float[camNum];

        for (int frame = 0; frame < totalKeyFrame; frame++)
        {
            for (int i = 0; i < camNum; i++)
            {
                var posTmp = path[i][frame];
                camValueList[i] += downsamplePixelBlend[frame][(int)posTmp.x, (int)posTmp.y];
                if (ifCloseCameraWindow[i][frame])
                {
                    continue;
                }
                for (int j = i + 1; j < camNum; j++)
                {
                    if (ifCloseCameraWindow[j][frame])
                    {
                        continue;
                    }
                    int   overlapBeginFrame = frame;
                    int   overlapEndFrame   = frame - 1;
                    float sumI = 0;
                    float sumJ = 0;
                    for (int k = frame; k < totalKeyFrame; k++)
                    {
                        var posI        = path[i][k];
                        var rectI       = rectOfPixel[(int)posI.x, (int)posI.y];
                        var singleAreaI = OpticalFlowCamerasController.GetRectWidth(rectI) * OpticalFlowCamerasController.GetRectHeight(rectI);
                        var posJ        = path[j][k];
                        var rectJ       = rectOfPixel[(int)posJ.x, (int)posJ.y];
                        var singleAreaJ = OpticalFlowCamerasController.GetRectWidth(rectJ) * OpticalFlowCamerasController.GetRectHeight(rectJ);
                        var overlapArea = opticalFlowCamerasController.GetRectRectOverlapArea(rectI, rectJ);
                        var iou         = overlapArea / (singleAreaI + singleAreaJ - overlapArea);
                        if (iou >= closeCameraWindowIouThreshold)
                        {
                            sumI           += downsamplePixelBlend[k][(int)posI.x, (int)posI.y];
                            sumJ           += downsamplePixelBlend[k][(int)posJ.x, (int)posJ.y];
                            overlapEndFrame = k;
                        }
                        else
                        {
                            break;
                        }
                    }
                    int closeCamId = sumI < sumJ ? i : j;
                    for (int k = overlapBeginFrame; k <= overlapEndFrame; k++)
                    {
                        ifCloseCameraWindow[closeCamId][k] = true;
                    }
                }
            }
        }

        float tmpMax = -1;

        for (int i = 0; i < camValueList.Length; i++)
        {
            if (camValueList[i] > tmpMax)
            {
                tmpMax            = camValueList[i];
                mostValuableCamId = i;
            }
        }

        for (int frame = 0; frame < totalKeyFrame; frame++)
        {
            for (int i = 0; i < camNum; i++)
            {
                if (ifCloseCameraWindow[i][frame])
                {
                    continue;
                }
                var  posI = path[i][frame];
                bool ifUpdateCloseCameraWindow  = false;
                int  frameAfterLastIdelKeyFrame = frame;
                //Debug.Log("posI: " + posI);
                if (downsamplePixelBlend[frame][(int)posI.x, (int)posI.y] >= minimalMeaningfulPixelSaliency || regionalSaliency[frame][(int)posI.x, (int)posI.y] >= minimalMeaningfulRegionalSaliency)
                {
                    ifUpdateCloseCameraWindow = true;
                }
                else if (frame == totalKeyFrame - 1)
                {
                    ifUpdateCloseCameraWindow  = true;
                    frameAfterLastIdelKeyFrame = totalKeyFrame;
                }
                if (ifUpdateCloseCameraWindow)
                {
                    float idleTime = ((float)frameAfterLastIdelKeyFrame - frameBeforeInitialIdleKeyFrame[i] - 1) / totalKeyFrame * videoPlayer.frameCount / videoPlayer.frameRate;
                    if (idleTime > allowedMaxIdleTime)
                    {
                        for (int j = frameBeforeInitialIdleKeyFrame[i] + 1; j < frameAfterLastIdelKeyFrame; j++)
                        {
                            ifCloseCameraWindow[i][j] = true;
                        }
                    }
                    frameBeforeInitialIdleKeyFrame[i] = frameAfterLastIdelKeyFrame;
                }
            }
        }

        for (int frame = 0; frame < totalKeyFrame; frame++)
        {
            int   closedCamCnt = 0;
            int   chosenCamId  = 0;
            float maxV         = 0;
            for (int camId = 0; camId < camNum; camId++)
            {
                if (ifCloseCameraWindow[camId][frame])
                {
                    closedCamCnt++;
                }
                var pos  = path[camId][frame];
                var nowV = Mathf.Max(downsamplePixelBlend[frame][(int)pos.x, (int)pos.y], regionalSaliency[frame][(int)pos.x, (int)pos.y]);
                if (maxV < nowV)
                {
                    maxV        = nowV;
                    chosenCamId = camId;
                }
            }
            if (closedCamCnt == camNum)
            {
                //Debug.Log(string.Format("ifCloseCameraWindow.count: {0}, frame: {1}, totalKeyFrame: {2}", ifCloseCameraWindow.Length, frame, totalKeyFrame));
                //Debug.Log("ifCloseCameraWindow[chosenCamId].Count: " + ifCloseCameraWindow[chosenCamId].Count);
                ifCloseCameraWindow[chosenCamId][frame] = false;
            }
        }
        for (int camId = 0; camId < camNum; camId++)
        {
            //Debug.Log("Camera" + camId + ":");
            for (int frame = 0; frame < totalKeyFrame; frame++)
            {
                var pos = path[camId][frame];
                //Debug.Log(string.Format("blendPixelSaliency: {0}, regionalSaliency: {1}", downsamplePixelBlendSaliencyMask[frame][(int)pos.x, (int)pos.y], regionalSaliency[frame][(int)pos.x, (int)pos.y]));
            }
            //Debug.Log(".................");
        }
        Debug.Log("ifCloseCameraWindow[0].Count: " + ifCloseCameraWindow[0].Count);
        Debug.Log("......................");
    }
    //Which camera windows of which frames are ready to close
    public void PrepareIfCloseCameraWindow()
    {
        var path = opticalFlowCamerasController.smoothPath;

        if (path == null)
        {
            Debug.Log("No smooth path");
            return;
        }
        InitDownsamplePixelBlendSaliencyMaskAndOpticalFlow();
        var camNum = panelVideoController.cameraGroupNum;
        var rectOfPixel = opticalFlowCamerasController.rectOfPixel;
        int downsampleWidth, downsampleHeight;

        opticalFlowCamerasController.GetDownsampleSize(out downsampleWidth, out downsampleHeight);
        var regionalSaliency = opticalFlowCamerasController.regionalSaliency;

        ifCloseCameraWindow = new List <bool> [camNum];
        var totalKeyFrame = path[0].Count;

        for (int i = 0; i < camNum; i++)
        {
            ifCloseCameraWindow[i] = new List <bool>();
        }

        //Which is the previous frame of the idle frame
        var frameBeforeInitialIdleKeyFrame = new int[camNum];

        for (int camId = 0; camId < camNum; camId++)
        {
            frameBeforeInitialIdleKeyFrame[camId] = -1;
        }
        for (int frame = 0; frame < totalKeyFrame; frame++)
        {
            for (int i = 0; i < camNum; i++)
            {
                ifCloseCameraWindow[i].Add(false);
            }
            for (int i = 0; i < camNum; i++)
            {
                var  posI = path[i][frame];
                bool ifUpdateCloseCameraWindow  = false;
                int  frameAfterLastIdelKeyFrame = frame;
                if (downsamplePixelBlendSaliencyMask[frame][(int)posI.x, (int)posI.y] >= minimalMeaningfulPixelSaliency || regionalSaliency[frame][(int)posI.x, (int)posI.y] >= minimalMeaningfulRegionalSaliency)
                {
                    ifUpdateCloseCameraWindow = true;
                }
                else if (frame == totalKeyFrame - 1)
                {
                    ifUpdateCloseCameraWindow  = true;
                    frameAfterLastIdelKeyFrame = totalKeyFrame;
                }
                if (ifUpdateCloseCameraWindow)
                {
                    float idleTime = ((float)frameAfterLastIdelKeyFrame - frameBeforeInitialIdleKeyFrame[i] - 1) / totalKeyFrame * videoPlayer.frameCount / videoPlayer.frameRate;
                    if (idleTime > allowedMaxIdleTime)
                    {
                        for (int j = frameBeforeInitialIdleKeyFrame[i] + 1; j < frameAfterLastIdelKeyFrame; j++)
                        {
                            ifCloseCameraWindow[i][j] = true;
                        }
                    }
                    frameBeforeInitialIdleKeyFrame[i] = frameAfterLastIdelKeyFrame;
                }
            }

            for (int i = 0; i < camNum; i++)
            {
                if (ifCloseCameraWindow[i][frame])
                {
                    continue;
                }
                var posI        = path[i][frame];
                var rectI       = rectOfPixel[(int)posI.x, (int)posI.y];
                var singleAreaI = OpticalFlowCamerasController.GetRectWidth(rectI) * OpticalFlowCamerasController.GetRectHeight(rectI);
                for (int j = i + 1; j < camNum; j++)
                {
                    if (ifCloseCameraWindow[j][frame])
                    {
                        continue;
                    }
                    var posJ        = path[j][frame];
                    var rectJ       = rectOfPixel[(int)posJ.x, (int)posJ.y];
                    var singleAreaJ = OpticalFlowCamerasController.GetRectWidth(rectJ) * OpticalFlowCamerasController.GetRectHeight(rectJ);
                    var overlapArea = opticalFlowCamerasController.GetRectRectOverlapArea(rectI, rectJ);
                    var iou         = overlapArea / (singleAreaI + singleAreaJ - overlapArea);
                    if (iou >= closeCameraWindowIouThreshold)
                    {
                        if (downsamplePixelBlendSaliencyMask[frame][(int)posI.x, (int)posI.y] < downsamplePixelBlendSaliencyMask[frame][(int)posJ.x, (int)posJ.y])
                        {
                            ifCloseCameraWindow[i][frame] = true;
                        }
                        else
                        {
                            ifCloseCameraWindow[j][frame] = true;
                        }
                    }
                }
            }
        }
        //Make sure that at least one camera is on for each frame
        for (int frame = 0; frame < totalKeyFrame; frame++)
        {
            int   closedCamCnt = 0;
            int   chosenCamId  = 0;
            float maxV         = 0;
            for (int camId = 0; camId < camNum; camId++)
            {
                if (ifCloseCameraWindow[camId][frame])
                {
                    closedCamCnt++;
                }
                var pos  = path[camId][frame];
                var nowV = Mathf.Max(downsamplePixelBlendSaliencyMask[frame][(int)pos.x, (int)pos.y], regionalSaliency[frame][(int)pos.x, (int)pos.y]);
                if (maxV < nowV)
                {
                    maxV        = nowV;
                    chosenCamId = camId;
                }
            }
            if (closedCamCnt == camNum)//Restore the most meaningful camera
            {
                ifCloseCameraWindow[chosenCamId][frame] = false;
            }
        }
        for (int camId = 0; camId < camNum; camId++)
        {
            Debug.Log("Camera" + camId + ":");
            for (int frame = 0; frame < totalKeyFrame; frame++)
            {
                var pos = path[camId][frame];
                Debug.Log(string.Format("blendPixelSaliency: {0}, regionalSaliency: {1}", downsamplePixelBlendSaliencyMask[frame][(int)pos.x, (int)pos.y], regionalSaliency[frame][(int)pos.x, (int)pos.y]));
            }
            Debug.Log(".................");
        }
        Debug.Log("ifCloseCameraWindow[0].Count: " + ifCloseCameraWindow[0].Count);
        Debug.Log("......................");
    }