// Project a 3D coordinate onto this view's image plane. The returned point is in the
// coordinate system of the image plane, with the origin at the top-left. The result will
// be fractional if the world coordinate doesn't project precisely onto the center of a
// pixel. If the z-coordinate of the passed point is zero, the returned point will have +/-
// infinity coordinate values.
public Point2Df ProjectFromWorld(Point3Df ptToProject)
{
Point2Df projectedPt =
((Point2Df)ptToProject * (DistanceToImagePlane / ptToProject.Z));
return(ConvertFromWorld(projectedPt));
}
// Get a depth map from 1080p post-processed images
public static double[][] GetDepthMap_1080()
{
int imgSize = 1080;
Point3Df orientation = new Point3Df(0, 0, 1);
double fov = 60.0 / 180.0 * Math.PI;
List <SceneView> views = new List <SceneView>();
foreach (string filename in Directory.GetFiles("Images/Rect/"))
{
if (!filename.EndsWith(".bmp"))
{
continue;
}
Bitmap bmp = new Bitmap(filename);
Image img = new Image(bmp);
Debug.Assert(img.Width == imgSize && img.Height == imgSize);
string xy = String.Concat(filename.SkipWhile(ch => ch != 'X'));
string[] x_y = String.Concat(xy.Take(xy.Length - ".bmp".Length)).
Split(new string[] { ", " }, StringSplitOptions.None);
double x = double.Parse(String.Concat(x_y[0].Skip(4)));
double y = double.Parse(String.Concat(x_y[1].Skip(4)));
Point3Df focalPt = new Point3Df(x * 10, y * 10, 0);
views.Add(new SceneView(img, focalPt, orientation, fov, fov));
}
return(DepthEstimator.CalculateDepthMap(views.First(), views.Skip(1).ToArray()));
}
public double Dot(Point3Df other)
{
double t1 = this.X * other.X;
double t2 = this.Y * other.Y;
double t3 = this.Z * other.Z;
return(t1 + t2 + t3);
}
public Point3Df Cross(Point3Df other)
{
double rx = this.Y * other.Z - this.Z * other.Y;
double ry = this.Z * other.X - this.X * other.Z;
double rz = this.X * other.Y - this.Y * other.X;
return(new Point3Df(rx, ry, rz));
}
// Returns the absolute difference between the passed images at the passed sub-pixel
// locations
public static double Compare(Image img1, Point2Df pt1, Image img2, Point2Df pt2)
{
Point3Df img1PixVal = img1.GetSubPixelValue(pt1);
Point3Df img2PixVal = img2.GetSubPixelValue(pt2);
return(Math.Abs(img1PixVal.X - img2PixVal.X) +
Math.Abs(img1PixVal.Y - img2PixVal.Y) +
Math.Abs(img1PixVal.Z - img2PixVal.Z));
}
public Point3Df Rotate(Point3Df axis, double angle)
{
Point3Df axisUnit = axis.Normalize();
Point3Df t1 = this * Math.Cos(angle);
Point3Df t2 = axisUnit.Cross(this) * Math.Sin(angle);
Point3Df t3 = axisUnit * (axisUnit.Dot(this)) * (1 - Math.Cos(angle));
return(t1 + t2 + t3);
}
// Given a pixel in this view and an assumed depth of the 3D point represented by that
// pixel, returns the point in the given view where the same 3D point will be represented
public Point2Df FindInOtherView(Point2Df pixel, SceneView target, double depth)
{
// Project pixel into 3D space
Point3Df worldPt = this.ProjectToWorld(pixel, depth);
// Transform the point into the target view's coordinate system
Point3Df targetPt = this.TransformToView(worldPt, target);
// Project the transformed point into the target view and return the result
return(target.ProjectFromWorld(targetPt));
}
// Projects a pixel in this view to world coordinates at the passed depth. Depth is
// measured by z-coordinate. The passed pixel may be fractional, and it does not have to be
// within the FOV of this view.
public Point3Df ProjectToWorld(Point2Df ptToProject, double depth)
{
// The 3D position of the pixel in question, in the coordinate system of the camera
Point3Df projectedPt = ConvertToWorld3D(ptToProject);
// Projection scale factor (for projecting out into the 3D space along a line of
// possible depths).
double projFactor = depth / projectedPt.Z;
// The 3D position of a point far along the line projected through the pixel being
// considered. It is at a maximum depth of maxDepth.
return(projectedPt * projFactor);
}
public static SceneView[][] CreateViewArrayFromBitmap(Bitmap lytroBmp)
{
Debug.Assert(lytroBmp == null ||
(lytroBmp.Height == Constants.ImgHeight && lytroBmp.Width == Constants.ImgWidth),
"Image dimensions don't match Lytro constants");
SceneView[][] sceneArr =
new SceneView[Constants.usedArrayHeight][];
for (int y = Constants.usedArrayStartY; y < Constants.usedArrayHeight + 1; y++)
{
SceneView[] sceneArrRow = new SceneView[Constants.usedArrayWidth];
double yOffset = Constants.SubImgSpacing *
((double)(Constants.usedArrayHeight - 1) / 2 - (y - Constants.usedArrayStartY));
for (int x = Constants.usedArrayStartX; x < Constants.usedArrayWidth + 1; x++)
{
Rectangle subImgROI =
new Rectangle(x * Constants.SubImgWidth, y * Constants.SubImgHeight,
Constants.SubImgWidth, Constants.SubImgHeight);
Image subImage;
if (lytroBmp == null)
{
subImage = new Image(Constants.SubImgWidth, Constants.SubImgHeight,
Constants.ImgChannels);
}
else
{
byte[] subImageData = Image.BitmapToBytes(lytroBmp, subImgROI);
subImage = new Image(subImageData, Constants.SubImgWidth,
Constants.SubImgHeight, Constants.ImgChannels);
}
double xOffset = Constants.SubImgSpacing *
((x - Constants.usedArrayStartX) - (double)(Constants.usedArrayWidth - 1) / 2);
Point3Df cop = new Point3Df(xOffset, yOffset, 0);
SceneView scene = new SceneView(subImage, cop, Constants.Orientation,
Constants.FieldOfView, Constants.FieldOfView);
sceneArrRow[x - Constants.usedArrayStartX] = scene;
}
sceneArr[y - Constants.usedArrayStartY] = sceneArrRow;
}
return(sceneArr);
}
public override bool Equals(object obj)
{
if (obj == null)
{
return(false);
}
Point3Df other = obj as Point3Df;
if ((object)other == null)
{
return(false);
}
return((this.X == other.X) && (this.Y == other.Y) && (this.Z == other.Z));
}
public SceneView(Image img, Point3Df centerOfProjection,
Point3Df orientation, double verticalFOV, double horizontalFOV)
{
this.img = img;
this.centerOfProjection = centerOfProjection;
this.orientation = orientation;
this.fieldOfView = new Point2Df(horizontalFOV, verticalFOV);
// CHECKME check vertical FOV vs horizontal FOV. Horizontal pixels are often a different
// CHECKME height - 1?
// size than vertical pixels, but we're trying to use them as our unit in 3D space as
// well so we have to make sure we're consistent.
// Precompute this distance based on the image size and FOV
this.distanceToImagePlane = ((double)img.Height / 2) / Math.Tan(verticalFOV / 2);
}
public Point3Df TransformToView(Point3Df ptToTransform, SceneView target)
{
// Translate
Point3Df transformed =
ptToTransform + this.CenterOfProjection - target.CenterOfProjection;
// Rotate
if (this.Orientation != target.Orientation)
{
Point3Df axis = target.Orientation.Cross(this.Orientation);
double angle = Math.Acos(this.Orientation.Dot(target.Orientation) /
(this.Orientation.Length() * target.Orientation.Length()));
transformed = transformed.Rotate(axis, angle);
}
return(transformed);
}
// Get a depth map from a pair of binocular test images
public static double[][] GetDepthMap_Binocular()
{
Bitmap leftBmp = new Bitmap("Images/im2.bmp");
Bitmap rightBmp = new Bitmap("Images/im6.bmp");
Image leftImg = new Image(leftBmp);
Image rightImg = new Image(rightBmp);
Debug.Assert(leftImg.Width == rightImg.Width && leftImg.Height == rightImg.Height);
Point3Df leftFocalPt = new Point3Df(-100, 0, 0);
Point3Df rightFocalPt = -leftFocalPt;
Point3Df orientation = new Point3Df(0, 0, 1);
double fov = 70.0 / 180.0 * Math.PI;
SceneView leftView = new SceneView(leftImg, leftFocalPt, orientation, fov, fov);
SceneView rightView = new SceneView(rightImg, rightFocalPt, orientation, fov, fov);
double[][] depthBuffer = DepthEstimator.CalculateDepthMap(leftView, rightView);
// Take the log of every depth. This helps smooth out noise
for (int y = 0; y < depthBuffer.Length; y++)
{
for (int x = 0; x < depthBuffer[0].Length; x++)
{
double depth = depthBuffer[y][x];
if (depth != DepthEstimator.Constants.InvalidDepth &&
!double.IsInfinity(depth))
{
depthBuffer[y][x] = Math.Log(depth);
}
}
}
return(depthBuffer);
}
public double Distance(Point3Df other)
{
return(Math.Sqrt(Math.Pow(this.X - other.X, 2) + Math.Pow(this.Y - other.Y, 2)
+ Math.Pow(this.Z - other.Z, 2)));
}
