迭代相机校准 - 没有收敛

计算科学 算法 Python 图像处理 计算机视觉 3d
2021-12-05 19:27:56

我正在尝试实现Datta 等人的研究论文“使用控制点的迭代细化的精确相机校准”中的算法。

运行多次迭代不会显示相机参数的收敛。

该算法定义为：

  1. 从一组 N 个图像中检测校准图案控制（也称为图像点）点（棋盘角、圆或环中心）。
  2. 使用这些控制点估计相机参数（内部和外部）。
  3. 循环直到收敛
    a。使用估计的相机参数对输入图像进行非失真处理，然后将未失真的图像重新投影到规范模式。
    湾。从规范模式本地化控制点。
    C。使用先前估计的相机参数重新投影新的控制点。
    d。使用新的投影点再估计一次相机参数。

我正在为这个实现使用 OpenCV 和 Python。我对前两个步骤使用标准相机校准程序。结果，我得到了相机参数的初步估计。

  • K：本征矩阵 [3x3]
  • D：镜头畸变系数[1x5]
  • rvec: 旋转向量 [Nx3]
  • T：平移向量[Nx3]

我看到的问题是焦距和主点坐标缓慢发散，相机校准的 RMS 误差没有按预期减少。例如：

初始化步骤
RMS = 0.2353278818397754
Fx的= 1436.241，FY = 1435.402，CX = 597.9362中，Cy = 342.601，DK1 = 0.065，DK2 = 0.172
迭代步骤1
RMS = 0.24574215365963115
Fx的= 1449.949，FY = 1446.909，CX = 597.509中，Cy = 341.320， DK1 = 0.050，DK2 = 0.556
迭代步骤2
RMS = 0.2451931123439023
FY = 1468.206，CX = 595.064，CY = 331.250，DK1 = 0.043，DK2 = 0.731
迭代步骤3
RMS = 0.24178717443873096 FX
= 1498.545，FY = 1489.129，FY = 1489.129， CX = 595.771，CY = 320.921，DK1 = 0.044，DK2 = 0.773
迭代步骤4
rms = 0.24339333469368066 FX
= 1526.091，FY = 1513.149，CX = 595.340，CY = 308.466，DK1 = 0.040，DK2 = 0.919
迭代步骤5
RMS = 0.24257897963694885
Fx=1559.688, Fy=1543.050, Cx=595.479, Cy=293.856, Dk1=0.045, Dk2=0.968

我已经检查了每个步骤，但我不知道我错在哪里。

请在下面找到我的代码。

import cv2 as cv
import numpy as np


# Find corners to remap the image to canonical pattern
def canonique(fname, corner_qty, criteria): 
    img = cv.imread(fname)
    grayscale = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
    image_size = grayscale.shape[::-1]
    ret, old_corners = cv.findChessboardCorners(grayscale, corner_qty, None)

    if ret is True:
        cv.cornerSubPix(grayscale, old_corners, (11, 11), (-1, -1), criteria)
        old_corners = np.asarray(old_corners).reshape(-1, 2)
        temp = np.zeros((old_corners.shape[0], 1))
        old_corners = np.hstack((old_corners, temp))
        old_corners = np.asarray(old_corners).astype('float32')

        af_trans = []
        width_step = float(image_size[0]) / float(corner_qty[0] + 1)
        height_step = float(image_size[1]) / float(corner_qty[1] + 1)
        af_trans.append([width_step, height_step])
        af_trans.append([image_size[0] - width_step, height_step])
        af_trans.append([width_step, image_size[1] - height_step])
        af_trans.append([image_size[0] - width_step, image_size[1] - height_step])
        af_trans = np.asarray(af_trans).astype('float32')

        bf_trans = [old_corners[0, 0:2],                                       # top-left point
                    old_corners[corner_qty[0] - 1, 0:2],                       # top-right point
                    old_corners[(corner_qty[1] - 1) * corner_qty[0], 0:2],     # bottom-left point
                    old_corners[corner_qty[1] * corner_qty[0] - 1, 0:2]]       # bottom-right point
        bf_trans = np.asarray(bf_trans).astype('float32')

        trans = cv.getPerspectiveTransform(bf_trans, af_trans)
        result = cv.warpPerspective(grayscale, trans, image_size)
        cv.imwrite(fname, result)


# Find corners in the list of images
def find_corners(filename, criteria, ctrl, corner_size, export_result=False, base_folder=''):
    image_size_read = (0, 0)
    control_pts, projected_pts = [], []

    for index, fname in enumerate(filename):
        img = cv.imread(fname)
        gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
        image_size_read = gray.shape[::-1]
        ret_find, corners = cv.findChessboardCorners(gray, corner_size, None)

        if ret_find is True:
            control_pts.append(ctrl)
            cv.cornerSubPix(gray, corners, (11, 11), (-1, -1), criteria)
            projected_pts.append(corners)

            if export_result is True:
                cv.drawChessboardCorners(img, corner_size, corners, ret_find)
                cv.imwrite(base_folder + '/ID{}FindCorner.png'.format(index + 1), img)

    return image_size_read, np.asarray(control_pts), projected_pts


def get_homography(k_matrix, r_vector, t_vector):
    r_matrix, _ = cv.Rodrigues(r_vector)
    hr = np.dot(k_matrix, np.dot(np.transpose(r_matrix), np.linalg.inv(k_matrix)))

    c = -np.dot(np.transpose(r_matrix), t_vector)
    u0 = -np.dot(k_matrix, c) / c[2]
    u = np.zeros((3, 1))

    u[0] = u0[0]
    u[1] = u0[1]
    u[2] = -u0[2]

    ht = np.array([[1, 0], [0, 1], [0, 0]])
    ht = np.hstack((ht, u))
    h = np.dot(ht, hr)

    return h


def normalize_control_points(ctrl_pts, image_res, corner_qty, norm_length):
    # Process the list of control points in each image
    for item, pt in enumerate(ctrl_pts):
        # the points are organized in the list by n_columns of n_rows points
        # We have
        #   corner_size[0] = n_rows
        #   corner_size[1] = n_columns
        pt[:, 0] *= float(corner_qty[0] + 1) * norm_length / image_res[0]
        pt[:, 1] *= float(corner_qty[1] + 1) * norm_length / image_res[1]
        ctrl_pts[item] = pt

    return ctrl_pts


corner_size = (9, 6)  # Quantity of corners on the checker board
corner_length = 1.0
iteration_quantity = 20
image_quantity = 9
res_folder = './Results'

# prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0), ...
control = np.zeros((corner_size[0] * corner_size[1], 3), np.float32)
control[:, :2] = np.mgrid[1: corner_size[0] + 1,
                          1: corner_size[1] + 1].T.reshape(-1, 2) * corner_length

TERMINATION_CRITERIA = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 30, 0.001)
file = []

# There should be image_quantity files for the camera
for i in range(image_quantity):
    file.append('./CheckerBoard-Images/checker_board_{:04d}_right.png'.format(i + 1))

# Process all of them to retrieve the checker boards corners.
# This is the initialization step of the iterative algorithm
image_size, control_points, projected_points = find_corners(file,
                                                            TERMINATION_CRITERIA,
                                                            control,
                                                            corner_size,
                                                            True,
                                                            res_folder)

control_points = np.asarray(control_points).astype('float32')
projected_points = np.asarray(projected_points).astype('float32')

rms, K, D, rvec, T = cv.calibrateCamera(control_points,
                                        projected_points,
                                        image_size,
                                        None, None)
rvec = np.asarray(rvec)
T = np.asarray(T)

# Compute the re-projection error to serve as baseline for further computation
print('Initialization Step')
print('Calibration RMS= ', rms)
print('Fx={}, Fy={}, Cx={}, Cy={}, Dk1={}, Dk2={}'.format(K[0, 0],
                                                          K[1, 1],
                                                          K[0, 2],
                                                          K[1, 2],
                                                          D[0, 0],
                                                          D[0, 1]))

# From the research paper "Accurate Camera Calibration using Iterative
# Refinement of Control Points", the following steps are
# LOOP START
# 1/ Un-distort and un-project: Use M/D/R/T to project the input images
#                               to a canonical pattern after removing distortions
# 2/ Localize control points: Localize calibration pattern control points
#                             in the canonical pattern
#
# 3/ Re-project: Project the control points using the estimated camera parameters
# 4/ Parameter Fitting: Use the projected control points to refine the camera
#                       parameters using Levenberg-Marquardt
# Note: THE CONVERGENCE CRITERIA IS NOT YET DEFINED

for iteration in range(iteration_quantity):
    iter_filename = []

    ################################################################################################
    # 1/ Un-distort and un-project
    for i in range(image_quantity):
        my_image = cv.imread(file[i])
        undist = cv.undistort(my_image, K, D)
        name = res_folder + '/ID{}UndistortIT{}.png'.format(i + 1, iteration)
        cv.imwrite(name, undist)

        H = get_homography(K, rvec[i], T[i])
        result = cv.warpPerspective(undist, H, (1280, 720), flags=cv.INTER_LINEAR)

        # For each image, we take the four "out" corners
        # Then we re-project them on a perfect mapping

        name = res_folder + '/ID{}UnprojectIT{}.png'.format(i + 1, iteration)
        cv.imwrite(name, result)
        iter_filename.append(name)
        canonique(name, corner_size, TERMINATION_CRITERIA)

    ################################################################################################
    # 2a/ Find the location of the new control points
    iter_ctrl_points = []
    iter_proj_points = []

    for index, fname in enumerate(iter_filename):
        img = cv.imread(fname)
        grayscale = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
        image_size_read = grayscale.shape[::-1]
        ret, corners = cv.findChessboardCorners(grayscale, corner_size, None)

        if ret is True:
            cv.cornerSubPix(grayscale, corners, (11, 11), (-1, -1), TERMINATION_CRITERIA)
            cv.drawChessboardCorners(img, corner_size, corners, ret)
            cv.imwrite(res_folder + '/ID{}NewpointsIT{}.png'.format(index + 1, iteration), img)

            corners = np.asarray(corners).reshape(-1, 2)
            temp = np.zeros((corners.shape[0], 1))
            corners = np.hstack((corners, temp))
        else:
            print('Error (It={}) on image {}'.format(iteration + 1, index + 1))

        iter_ctrl_points.append(corners)

    iter_ctrl_points = np.asarray(iter_ctrl_points).astype('float32')

    # 2b/ Normalization of the control points
    iter_ctrl_points = normalize_control_points(iter_ctrl_points,
                                                image_size,
                                                corner_size,
                                                corner_length)

    ################################################################################################
    # 3/ Project the new control points
    for i in range(len(iter_ctrl_points)):
        proj_points, _ = cv.projectPoints(iter_ctrl_points[i], rvec[i], T[i], K, D)
        iter_proj_points.append(proj_points)

    ################################################################################################
    # 4/ Parameter fitting (new camera calibration)
    rms, K, D, rvec, T = cv.calibrateCamera(control_points,
                                            iter_proj_points,
                                            image_size, K, D)

    rvec = np.asarray(rvec)
    T = np.asarray(T)
    print('Iterative Step {}'.format(iteration + 1))
    print('Calibration RMS= ', rms)
    print('Fx={}, Fy={}, Cx={}, Cy={}, Dk1={}, Dk2={}'.format(K[0, 0],
                                                              K[1, 1],
                                                              K[0, 2],
                                                              K[1, 2],
                                                              D[0, 0],
                                                              D[0, 1]))

这是我正在使用的输入图片的示例。 在此处输入图像描述

1个回答

我必须告诉你。

我实现了这个算法（甚至与其他有经验的人仔细检查），但是没有运气。我想，这对作者有用，但根据我们的数据，确实没有任何改善。

此外，曾几何时，作者提供了一个代码。我也在我自己的数据集上尝试过。再次，没有工作。因此，我现在确信此方法针对特定数据集进行了调整，并且不会一概而论。

相反，这是另一项工作，可能更有希望：

使用不完美的平面目标进行更准确的针孔相机校准 Klaus H. Strobl, Gerd Hirzinger

http://elib.dlr.de/71888/1/strobl_2011iccv.pdf

该算法可能在DLR 校准工具箱中实现。我建议从那里开始。

确实，更好的校准是人们让视力正常工作的一个严肃话题，并且没有建立获得最佳校准的明确程序。也许超出范围，但我建议检查摄影测量/遥感学者（例如 ISPRS）的作品，它们很少假设线性化相机模型。

其它你可能感兴趣的问题
上一篇超节点与多额因子分解 下一篇一个简单的 SPH 示例的实现