/// <summary> /// update stereo correspondence /// </summary> /// <param name="left_bmp_colour">rectified left colour image data</param> /// <param name="right_bmp_colour">rectified right colour image_data</param> /// <param name="left_bmp">rectified left colour image data</param> /// <param name="right_bmp">rectified right colour image_data</param> /// <param name="image_width">width of the image</param> /// <param name="image_height">height of the image</param> /// <param name="calibration_offset_x">offset calculated during camera calibration</param> /// <param name="calibration_offset_y">offset calculated during camera calibration</param> public override void Update( byte[] left_bmp_colour, byte[] right_bmp_colour, byte[] left_bmp, byte[] right_bmp, int image_width, int image_height, float calibration_offset_x, float calibration_offset_y) { // update sparse stereo features UpdateSimple(left_bmp_colour, right_bmp_colour, left_bmp, right_bmp, image_width, image_height, calibration_offset_x, calibration_offset_y); // extract FAST corner features from the left image FASTcorner[] corners_all = FAST.fast_corner_detect_9(left_bmp_mono[0], image_width, image_height, FAST_feature_threshold); FASTcorner[] corners = FAST.fast_nonmax(left_bmp_mono[0], image_width, image_height, corners_all, FAST_feature_threshold * 2, 0, 0); //features.Clear(); // adjust the threshold int no_of_feats = 0; if (corners != null) { no_of_feats = corners.Length; //for (int i = 0; i < corners.Length; i++) // features.Add(new StereoFeature(corners[i].x, corners[i].y, 20)); } if (no_of_feats < required_FAST_features / 2) { FAST_feature_threshold -= 1; } if (no_of_feats > required_FAST_features) { FAST_feature_threshold += 1; } if (Math.Abs(no_of_feats - required_FAST_features) < 100) { UpdateCornerDisparities(corners, 10); } else { features.Clear(); } //if (Math.Abs(no_of_feats - required_FAST_features) < 100) // lines = DetectLines(corners, left_bmp_mono[0], image_width, image_height); //else // lines = null; }
public void update(Byte[] left_bmp, Byte[] right_bmp, int wdth, int hght, int bytes_per_pixel, float calibration_offset_x, float calibration_offset_y, ref int threshold) { Byte[] left_img; Byte[] right_img; int max_disparity_pixels = wdth * max_disparity / 100; if (bytes_per_pixel == 1) { left_img = left_bmp; right_img = right_bmp; } else { left_img = new Byte[wdth * hght]; right_img = new Byte[wdth * hght]; int n = 0; for (int i = 0; i < wdth * hght * bytes_per_pixel; i += bytes_per_pixel) { int left_tot = 0; int right_tot = 0; for (int c = 0; c < bytes_per_pixel; c++) { left_tot += left_bmp[i + c]; right_tot += right_bmp[i + c]; } left_img[n] = (Byte)(left_tot / bytes_per_pixel); right_img[n] = (Byte)(right_tot / bytes_per_pixel); n++; } } // set a default threshold value if none is specified if (threshold == 0) { threshold = 50; } // extract features from the left image FASTcorner[] left_corners_all = FAST.fast_corner_detect_10(left_img, wdth, hght, threshold); FASTcorner[] left_corners = FAST.fast_nonmax(left_img, wdth, hght, left_corners_all, threshold * 2, 0, 0); // only continue if there aren't too many features no_of_selected_features = 0; if (left_corners != null) { if (left_corners.Length < required_features * 2) { // extract features from the right image FASTcorner[] right_corners_all = FAST.fast_corner_detect_10(right_img, wdth, hght, threshold); FASTcorner[] right_corners = FAST.fast_nonmax(right_img, wdth, hght, right_corners_all, threshold * 2, -calibration_offset_x, -calibration_offset_y); if (right_corners != null) { // update feature properties used for matching FAST.fast_update(left_corners, wdth / 5, hght / 5); FAST.fast_update(right_corners, wdth / 5, hght / 5); // adjust the threshold int no_of_feats = (left_corners.Length + right_corners.Length) / 2; if (no_of_feats < required_features / 2) { threshold -= 4; } if (no_of_feats > required_features) { threshold += 4; } // this is a test int n = 0; /* * for (int i = 0; i < left_corners.Length; i++) * { * FASTcorner corner = left_corners[i]; * selected_features[(n * 3)] = corner.x; * selected_features[(n * 3) + 1] = corner.y; * selected_features[(n * 3) + 2] = 3; * n++; * } * for (int i = 0; i < right_corners.Length; i++) * { * FASTcorner corner = right_corners[i]; * selected_features[(n * 3)] = corner.x; * selected_features[(n * 3) + 1] = corner.y; * selected_features[(n * 3) + 2] = 3; * n++; * } */ no_of_selected_features = n; // bucket the data into rows // this helps to make matching more efficient ArrayList[] left_row = new ArrayList[hght]; ArrayList[] right_row = new ArrayList[hght]; for (int i = 0; i < left_corners.Length; i++) { int index = left_corners[i].y; if (left_row[index] == null) { left_row[index] = new ArrayList(); } left_row[index].Add(left_corners[i]); } for (int i = 0; i < right_corners.Length; i++) { int index = right_corners[i].y; if (right_row[index] == null) { right_row[index] = new ArrayList(); } right_row[index].Add(right_corners[i]); } // match rows int vertical_search = 0; for (int y = 0; y < hght; y++) { ArrayList row = left_row[y]; if (row != null) { for (int f1 = 0; f1 < row.Count; f1++) { FASTcorner corner1 = (FASTcorner)row[f1]; int min_score = 60; float disp = -1; for (int yy = y - vertical_search; yy <= y + vertical_search; yy++) { if ((yy > -1) && (yy < hght)) { ArrayList row2 = right_row[yy]; if (row2 != null) { for (int f2 = 0; f2 < row2.Count; f2++) { FASTcorner corner2 = (FASTcorner)row2[f2]; int dx = corner1.x - corner2.x; if ((dx >= 0) && (dx < max_disparity_pixels)) { int dy = yy - y; int score = corner1.matching_score(corner2) * (Math.Abs(dy) + 1); //int score = Math.Abs(dy) + dx; if ((score > -1) && ((min_score == -1) || (score < min_score))) { min_score = score; float left_fx = 0, right_fx = 0, fy = 0; subPixelLocation(left_img, wdth, hght, corner1.x, corner1.y, 5, 1, ref left_fx, ref fy); subPixelLocation(right_img, wdth, hght, corner2.x, corner2.y, 5, 1, ref right_fx, ref fy); disp = left_fx - right_fx; } } } } } } if (disp > -1) { if ((corner1.x > -1) && (corner1.x < 320)) { selected_features[(no_of_selected_features * 3)] = corner1.x; selected_features[(no_of_selected_features * 3) + 1] = corner1.y; selected_features[(no_of_selected_features * 3) + 2] = disp; no_of_selected_features++; } } } } } } else { threshold -= 4; } } else { threshold += 4; } } else { threshold -= 4; } if (threshold < 10) { threshold = 10; } }
public void Update(Byte[] raw_image, int width, int height) { mono_image = sluggish.utilities.image.monoImage(raw_image, width, height); if (line_threshold == 0) { line_threshold = 200; } if (corner_threshold == 0) { corner_threshold = 50; } FASTcorner[] corners_all = FAST.fast_corner_detect_10(mono_image, width, height, corner_threshold); FASTcorner[] corners1 = FAST.fast_nonmax(mono_image, width, height, corners_all, corner_threshold * 2, 0, 0); corners = local_nonmax(corners1, width / 15, height / 15); if (corners != null) { int no_of_feats = corners1.Length; if (no_of_feats < required_features / 2) { corner_threshold -= 4; } if (no_of_feats > required_features) { corner_threshold += 4; } if ((no_of_feats > 1) && (no_of_feats < required_features * 2)) { int min_line_length = width / 10; lines = FAST.fast_lines(mono_image, width, height, corners, line_threshold, min_line_length); if (lines != null) { int no_of_lines = 0; for (int i = 0; i < lines.Length; i++) { FASTline line = lines[i]; if (!(((line.point1.y < height / 10) && (line.point2.y < height / 10)) || ((line.point1.y > height - (height / 20)) && (line.point2.y > height - (height / 20))) || ((line.point1.x > width - (width / 20)) && (line.point2.x > width - (width / 20))) || ((line.point1.x < width / 20) && (line.point2.x < width / 20)) )) { line.Visible = true; no_of_lines++; } } // detect the horizon if (showHorizon) { int vertical_position = 0; float gradient = FAST.horizon_detection(raw_image, width, height, lines, horizon_threshold, ref vertical_position); if (vertical_position > 0) { int tx = (width / 2) + (int)((width / 4) * 1); int ty = vertical_position - (int)((width / 4) * gradient); //ty = ty * height / width; sluggish.utilities.drawing.drawLine(raw_image, width, height, width / 2, vertical_position, tx, ty, 0, 255, 0, 1, false); } } // draw lines if (drawLines) { for (int i = 0; i < lines.Length; i++) { FASTline line = lines[i]; if (line.Visible) { if (!line.onHorizon) { sluggish.utilities.drawing.drawLine(raw_image, width, height, line.point1.x, line.point1.y, line.point2.x, line.point2.y, 255, 0, 0, 0, false); } else { sluggish.utilities.drawing.drawLine(raw_image, width, height, line.point1.x, line.point1.y, line.point2.x, line.point2.y, 0, 255, 0, 0, false); } } } } if (no_of_lines < required_lines) { line_threshold -= 4; } if (no_of_lines > required_lines) { line_threshold += 8; } if (line_threshold < 100) { line_threshold = 100; } // detect the gravity angle if (showGravity) { float gravity_angle = FAST.gravity_direction(lines); if (drawLines) { int pendulum_length = width / 8; int px = (width / 2) + (int)(pendulum_length * Math.Sin(gravity_angle)); int py = (height / 2) + (int)(pendulum_length * Math.Cos(gravity_angle) * height / width); sluggish.utilities.drawing.drawLine(raw_image, width, height, width / 2, height / 2, px, py, 0, 255, 0, 1, false); int arrow_length = pendulum_length / 5; float angle2 = gravity_angle + (float)(Math.PI * 0.8f); int px2 = px + (int)(arrow_length * Math.Sin(angle2)); int py2 = py + (int)(arrow_length * Math.Cos(angle2) * height / width); sluggish.utilities.drawing.drawLine(raw_image, width, height, px2, py2, px, py, 0, 255, 0, 1, false); angle2 = gravity_angle - (float)(Math.PI * 0.8f); px2 = px + (int)(arrow_length * Math.Sin(angle2)); py2 = py + (int)(arrow_length * Math.Cos(angle2) * height / width); sluggish.utilities.drawing.drawLine(raw_image, width, height, px2, py2, px, py, 0, 255, 0, 1, false); } } } } } //odometry.update(disp_bmp_data, width, height); output_image = raw_image; }