diff --git a/mycobot_280/mycobot_280arduino/CMakeLists.txt b/mycobot_280/mycobot_280arduino/CMakeLists.txt index bfc6079..00529f8 100644 --- a/mycobot_280/mycobot_280arduino/CMakeLists.txt +++ b/mycobot_280/mycobot_280arduino/CMakeLists.txt @@ -27,6 +27,7 @@ catkin_install_python(PROGRAMS scripts/listen_real.py scripts/listen_real_of_topic.py scripts/simple_gui.py + scripts/detect_stag.py DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} ) diff --git a/mycobot_280/mycobot_280arduino/camera_calibration/EyesInHand_matrix.json b/mycobot_280/mycobot_280arduino/camera_calibration/EyesInHand_matrix.json new file mode 100644 index 0000000..5db3d2b --- /dev/null +++ b/mycobot_280/mycobot_280arduino/camera_calibration/EyesInHand_matrix.json @@ -0,0 +1 @@ +[[0.7978586214708513, 0.59877174890628, -0.06995722161683607, -42.33432105525292], [0.6010697464630348, -0.7990332868561791, 0.016154453957772947, -35.01038302335407], [0.04622531807890511, 0.05493813982585287, 0.9974191800647204, 39.342478235028345], [0.0, 0.0, 0.0, 1.0]] \ No newline at end of file diff --git a/mycobot_280/mycobot_280arduino/camera_calibration/camera_calibration.py b/mycobot_280/mycobot_280arduino/camera_calibration/camera_calibration.py new file mode 100644 index 0000000..7f363eb --- /dev/null +++ b/mycobot_280/mycobot_280arduino/camera_calibration/camera_calibration.py @@ -0,0 +1,307 @@ +import cv2 +from uvc_camera import UVCCamera +import stag +import numpy as np +import json +import time +import os +from scipy.linalg import svd +from pymycobot import * +from marker_utils import * +import shutil +import glob + + +ports = glob.glob('/dev/ttyUSB*') + glob.glob('/dev/ttyACM*') + glob.glob('/dev/ttyAMA*') +print(ports) +if ports: + arm_port = ports[0] +else: + raise Exception("No MyCobot device found") + + +mc = MyCobot280(port=arm_port, baudrate=1000000) # 设置端口 + +np.set_printoptions(suppress=True, formatter={'float_kind': '{:.2f}'.format}) + + + +class camera_detect: + #Camera parameter initialize + def __init__(self, camera_id, marker_size, mtx, dist): + self.camera_id = camera_id + self.mtx = mtx + self.dist = dist + self.marker_size = marker_size + self.camera = UVCCamera(self.camera_id, self.mtx, self.dist) + self.camera_open() + + self.origin_mycbot_horizontal = [-90, -35.85, -52.91, 88.59, 0, 0.0] + self.origin_mycbot_level = [-90, 5, -104, 14, 0, 0] + + self.IDENTIFY_LEN = 300 + + # Initialize EyesInHand_matrix to None or load from a document if available + self.EyesInHand_matrix = None + file_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + self.matrix_file_path = os.path.join(file_dir, "config","EyesInHand_matrix.json") + + self.load_matrix() + + def save_matrix(self, filename="EyesInHand_matrix.json"): + # Save the EyesInHand_matrix to a JSON file + if self.EyesInHand_matrix is not None: + with open(filename, 'w') as f: + json.dump(self.EyesInHand_matrix.tolist(), f) + + try: + # 复制文件到目标路径 + shutil.copy(filename, self.matrix_file_path) + print(f"File copied to {self.matrix_file_path}") + except IOError as e: + print(f"Failed to copy file: {e}") + + def load_matrix(self, filename="EyesInHand_matrix.json"): + # Load the EyesInHand_matrix from a JSON file, if it exists + try: + with open(filename, 'r') as f: + self.EyesInHand_matrix = np.array(json.load(f)) + except FileNotFoundError: + print("Matrix file not found. EyesInHand_matrix will be initialized later.") + + def CvtRotationMatrixToEulerAngle(self, pdtRotationMatrix): + pdtEulerAngle = np.zeros(3) + pdtEulerAngle[2] = np.arctan2(pdtRotationMatrix[1, 0], pdtRotationMatrix[0, 0]) + fCosRoll = np.cos(pdtEulerAngle[2]) + fSinRoll = np.sin(pdtEulerAngle[2]) + pdtEulerAngle[1] = np.arctan2(-pdtRotationMatrix[2, 0], + (fCosRoll * pdtRotationMatrix[0, 0]) + (fSinRoll * pdtRotationMatrix[1, 0])) + pdtEulerAngle[0] = np.arctan2((fSinRoll * pdtRotationMatrix[0, 2]) - (fCosRoll * pdtRotationMatrix[1, 2]), + (-fSinRoll * pdtRotationMatrix[0, 1]) + (fCosRoll * pdtRotationMatrix[1, 1])) + return pdtEulerAngle + + # 将欧拉角转为旋转矩阵 + def CvtEulerAngleToRotationMatrix(self, ptrEulerAngle): + ptrSinAngle = np.sin(ptrEulerAngle) + ptrCosAngle = np.cos(ptrEulerAngle) + ptrRotationMatrix = np.zeros((3, 3)) + ptrRotationMatrix[0, 0] = ptrCosAngle[2] * ptrCosAngle[1] + ptrRotationMatrix[0, 1] = ptrCosAngle[2] * ptrSinAngle[1] * ptrSinAngle[0] - ptrSinAngle[2] * ptrCosAngle[0] + ptrRotationMatrix[0, 2] = ptrCosAngle[2] * ptrSinAngle[1] * ptrCosAngle[0] + ptrSinAngle[2] * ptrSinAngle[0] + ptrRotationMatrix[1, 0] = ptrSinAngle[2] * ptrCosAngle[1] + ptrRotationMatrix[1, 1] = ptrSinAngle[2] * ptrSinAngle[1] * ptrSinAngle[0] + ptrCosAngle[2] * ptrCosAngle[0] + ptrRotationMatrix[1, 2] = ptrSinAngle[2] * ptrSinAngle[1] * ptrCosAngle[0] - ptrCosAngle[2] * ptrSinAngle[0] + ptrRotationMatrix[2, 0] = -ptrSinAngle[1] + ptrRotationMatrix[2, 1] = ptrCosAngle[1] * ptrSinAngle[0] + ptrRotationMatrix[2, 2] = ptrCosAngle[1] * ptrCosAngle[0] + return ptrRotationMatrix + + def wait(self): + time.sleep(0.5) + while(mc.is_moving() == 1): + time.sleep(0.2) + + def camera_open(self): + self.camera.capture() # 打开摄像头 + + # 获取物体坐标(相机系) + def calc_markers_base_position(self, corners, ids): + if len(corners) == 0: + return [] + rvecs, tvecs = solve_marker_pnp(corners, self.marker_size, self.mtx, self.dist) # 通过二维码角点获取物体旋转向量和平移向量 + for i, tvec, rvec in zip(ids, tvecs, rvecs): + tvec = tvec.squeeze().tolist() + rvec = rvec.squeeze().tolist() + rotvector = np.array([[rvec[0], rvec[1], rvec[2]]]) + Rotation = cv2.Rodrigues(rotvector)[0] # 将旋转向量转为旋转矩阵 + Euler = self.CvtRotationMatrixToEulerAngle(Rotation) # 将旋转矩阵转为欧拉角 + target_coords = np.array([tvec[0], tvec[1], tvec[2], Euler[0], Euler[1], Euler[2]]) # 物体坐标(相机系) + return target_coords + + + def eyes_in_hand_calculate(self, pose, tbe, Mc, Mr): + + pose,Mr = map(np.array, [pose,Mr]) + # 将角度从度数转换为弧度 + euler = pose * np.pi / 180 + Rbe = self.CvtEulerAngleToRotationMatrix(euler) + Reb = Rbe.T + + A = np.empty((3, 0)) + b_comb = np.empty((3, 0)) + + r = tbe.shape[0] + + for i in range(1, r): + A = np.hstack((A, (Mc[i, :].reshape(3, 1) - Mc[0, :].reshape(3, 1)))) + b_comb = np.hstack((b_comb, (tbe[0, :].reshape(3, 1) - tbe[i, :].reshape(3, 1)))) + + b = Reb @ b_comb + U, _, Vt = svd(A @ b.T) + Rce = Vt.T @ U.T + + tbe_sum = np.sum(tbe, axis=0) + Mc_sum = np.sum(Mc, axis=0) + + tce = Reb @ (Mr.reshape(3, 1) - (1/r) * tbe_sum.reshape(3, 1) - (1/r) * (Rbe @ Rce @ Mc_sum.reshape(3, 1))) + tce[2] -= self.IDENTIFY_LEN #用于保持识别距离 + + EyesInHand_matrix = np.vstack((np.hstack((Rce, tce)), np.array([0, 0, 0, 1]))) + print("EyesInHand_matrix = ", EyesInHand_matrix) + return EyesInHand_matrix + + # 读取Camera坐标(单次) + def stag_identify(self): + self.camera.update_frame() # 刷新相机界面 + frame = self.camera.color_frame() # 获取当前帧 + (corners, ids, rejected_corners) = stag.detectMarkers(frame, 11) # 获取画面中二维码的角度和id + # 绘制检测到的标记及其ID + stag.drawDetectedMarkers(frame, corners, ids) + # 绘制被拒绝的候选区域,颜色设为红色 + stag.drawDetectedMarkers(frame, rejected_corners, border_color=(255, 0, 0)) + marker_pos_pack = self.calc_markers_base_position(corners, ids) # 获取物的坐标(相机系) + if(len(marker_pos_pack) == 0): + marker_pos_pack, ids = self.stag_identify() + # print("Camera coords = ", marker_pos_pack) + # cv2.imshow("rrrr", frame) + # cv2.waitKey(1) + return marker_pos_pack, ids + + def Matrix_identify(self, ml): + ml.send_angles(self.origin_mycbot_level, 50) # 移动到观测点 + self.wait() + input("make sure camera can observe the stag, enter any key quit") + coords = ml.get_coords() + pose = coords[3:6] + print(pose) + # self.camera_open_loop() + Mc1,tbe1,pos1 = self.reg_get(ml) + ml.send_coord(1, coords[0] + 50, 30) + self.wait() + Mc2,tbe2,pos2 = self.reg_get(ml) + ml.send_coord(3, coords[2] + 20, 30) + self.wait() + Mc3,tbe3,pos3 = self.reg_get(ml) + ml.send_coord(2, coords[1] + 20, 30) + self.wait() + Mc4,tbe4,pos4 = self.reg_get(ml) + ml.send_coord(1, coords[0] + 20, 30) + self.wait() + Mc5,tbe5,pos5 = self.reg_get(ml) + tbe = np.vstack([tbe1, tbe2, tbe3, tbe4, tbe5]) + Mc = np.vstack([Mc1, Mc2, Mc3, Mc4, Mc5]) + state = None + if self.EyesInHand_matrix is not None: + state = True + pos = np.vstack([pos1, pos2, pos3, pos4, pos5]) + r = pos.shape[0] + for i in range(1, r): + for j in range(3): + err = abs(pos[i][j] - pos[0][j]) + if(err > 10): + state = False + # print("matrix error") + return pose, tbe, Mc, state + + def Eyes_in_hand_calibration(self, ml): + ml.set_end_type(0) + pose, tbe, Mc, state = self.Matrix_identify(ml) + if(state == True): + print("Calibration Complete EyesInHand_matrix = ", self.EyesInHand_matrix) + return + + input("Move the end of the robot arm to the calibration point, press any key to release servo") + ml.release_all_servos() + input("focus servo and get current coords") + ml.power_on() + time.sleep(1) + coords = ml.get_coords() + while len(coords) == 0: + coords = ml.get_coords() + Mr = coords[0:3] + print(Mr) + + self.EyesInHand_matrix = self.eyes_in_hand_calculate(pose, tbe, Mc, Mr) + print("EyesInHand_matrix = ", self.EyesInHand_matrix) + self.save_matrix() # Save the matrix to a file after calculating it + print("save successe, wait to verify") + + pose, tbe, Mc, state = self.Matrix_identify(ml) + if state != True: + self.EyesInHand_matrix = self.eyes_in_hand_calculate(pose, tbe, Mc, Mr) + + def Transformation_matrix(self,coord): + """坐标转换为齐次变换矩阵 + + Args: + coord (_type_): (x,y,z,rx,ry,rz) + + Returns: + _type_: _description_ + """ + position_robot = coord[:3] + pose_robot = coord[3:] + # 将欧拉角转为旋转矩阵 + RBT = self.CvtEulerAngleToRotationMatrix(pose_robot) + PBT = np.array([[position_robot[0]], + [position_robot[1]], + [position_robot[2]]]) + temp = np.concatenate((RBT, PBT), axis=1) + array_1x4 = np.array([[0, 0, 0, 1]]) + # 将两个数组按行拼接起来 + matrix = np.concatenate((temp, array_1x4), axis=0) + return matrix + + def Eyes_in_hand(self, coord, camera, Matrix_TC): + Position_Camera = np.transpose(camera[:3]) # 相机坐标 + Matrix_BT = self.Transformation_matrix(coord) # 机械臂坐标矩阵 + + Position_Camera = np.append(Position_Camera, 1) # 物体坐标(相机系) + Position_B = Matrix_BT @ Matrix_TC @ Position_Camera # 物体坐标(基坐标系) + return Position_B + + def stag_robot_identify(self, ml): + marker_pos_pack,ids = self.stag_identify() + target_coords = ml.get_coords() # 获取机械臂当前坐标 + while (target_coords is None): + target_coords = ml.get_coords() + # print("current_coords", target_coords) + cur_coords = np.array(target_coords.copy()) + cur_coords[-3:] *= (np.pi / 180) # 将角度值转为弧度值 + fact_bcl = self.Eyes_in_hand(cur_coords, marker_pos_pack, self.EyesInHand_matrix) # 通过矩阵变化将物体坐标(相机系)转成(基坐标系) + + for i in range(3): + target_coords[i] = fact_bcl[i] + + return target_coords,ids + + def reg_get(self, ml): + target_coords = None + for i in range(30): + Mc_all,_ = self.stag_identify() + if self.EyesInHand_matrix is not None: + target_coords,_ = self.stag_robot_identify(ml) + + tbe_all = ml.get_coords() # 获取机械臂当前坐标 + while (tbe_all is None): + tbe_all = ml.get_coords() + + tbe = np.array(tbe_all[0:3]) + Mc = np.array(Mc_all[0:3]) + print("tbe = ", tbe) + print("Mc = ", Mc) + return Mc,tbe,target_coords + + +if __name__ == "__main__": + if mc.is_power_on()==0: + mc.power_on() + camera_params = np.load("camera_params.npz") # 相机配置文件 + mtx, dist = camera_params["mtx"], camera_params["dist"] + m = camera_detect(0, 32, mtx, dist) + # tool_len = 20 + # mc.set_tool_reference([0, 0, tool_len, 0, 0, 0]) + # mc.set_end_type(0) + + m.Eyes_in_hand_calibration(mc) #手眼标定 + diff --git a/mycobot_280/mycobot_280arduino/camera_calibration/camera_params.npz b/mycobot_280/mycobot_280arduino/camera_calibration/camera_params.npz new file mode 100644 index 0000000..70b136d Binary files /dev/null and b/mycobot_280/mycobot_280arduino/camera_calibration/camera_params.npz differ diff --git a/mycobot_280/mycobot_280arduino/camera_calibration/marker_utils.py b/mycobot_280/mycobot_280arduino/camera_calibration/marker_utils.py new file mode 100644 index 0000000..21c325b --- /dev/null +++ b/mycobot_280/mycobot_280arduino/camera_calibration/marker_utils.py @@ -0,0 +1,59 @@ +import cv2 +import numpy as np +import typing as T +from numpy.typing import NDArray, ArrayLike + + +class MarkerInfo(T.TypedDict): + corners: np.ndarray + tvec: np.ndarray + rvec: np.ndarray + num_id: int + + +def solve_marker_pnp(corners: NDArray, marker_size: int, mtx: NDArray, dist: NDArray): + """ + This will estimate the rvec and tvec for each of the marker corners detected by: + corners, ids, rejectedImgPoints = detector.detectMarkers(image) + corners - is an array of detected corners for each detected marker in the image + marker_size - is the size of the detected markers + mtx - is the camera matrix + distortion - is the camera distortion matrix + RETURN list of rvecs, tvecs, and trash (so that it corresponds to the old estimatePoseSingleMarkers()) + """ + marker_points = np.array( + [ + [-marker_size / 2, marker_size / 2, 0], + [marker_size / 2, marker_size / 2, 0], + [marker_size / 2, -marker_size / 2, 0], + [-marker_size / 2, -marker_size / 2, 0], + ], + dtype=np.float32, + ) + rvecs = [] + tvecs = [] + for corner in corners: + retval, rvec, tvec = cv2.solvePnP( + marker_points, + corner, + mtx, + dist, + flags=cv2.SOLVEPNP_IPPE_SQUARE, + ) + if retval: + rvecs.append(rvec) + tvecs.append(tvec) + + rvecs = np.array(rvecs) # type: ignore + tvecs = np.array(tvecs) # type: ignore + (rvecs - tvecs).any() # type: ignore + return rvecs, tvecs + + +def draw_marker(frame: np.ndarray, corners, tvecs, rvecs, ids, mtx, dist) -> None: + # cv2.aruco.drawDetectedMarkers(frame, corners, None, borderColor=(0, 255, 0)) + cv2.aruco.drawDetectedMarkers(frame, corners, ids, borderColor=(0, 200, 200)) + for i in range(len(ids)): + corner, tvec, rvec, marker_id = corners[i], tvecs[i], rvecs[i], ids[i] + cv2.drawFrameAxes(frame, mtx, dist, rvec, tvec, 60, 2) + diff --git a/mycobot_280/mycobot_280arduino/camera_calibration/readme.docx b/mycobot_280/mycobot_280arduino/camera_calibration/readme.docx new file mode 100644 index 0000000..841b691 Binary files /dev/null and b/mycobot_280/mycobot_280arduino/camera_calibration/readme.docx differ diff --git a/mycobot_280/mycobot_280arduino/camera_calibration/stag.png b/mycobot_280/mycobot_280arduino/camera_calibration/stag.png new file mode 100644 index 0000000..6745a0a Binary files /dev/null and b/mycobot_280/mycobot_280arduino/camera_calibration/stag.png differ diff --git a/mycobot_280/mycobot_280arduino/camera_calibration/uvc_camera.py b/mycobot_280/mycobot_280arduino/camera_calibration/uvc_camera.py new file mode 100644 index 0000000..d0d7ee4 --- /dev/null +++ b/mycobot_280/mycobot_280arduino/camera_calibration/uvc_camera.py @@ -0,0 +1,56 @@ +import cv2 +import numpy as np +import time +import typing + + +class UVCCamera: + def __init__( + self, + cam_index=0, + mtx=None, + dist=None, + capture_size: typing.Tuple[int, int] = (640, 480), + ): + super().__init__() + self.cam_index = cam_index + self.mtx = mtx + self.dist = dist + self.curr_color_frame: typing.Union[np.ndarray, None] = None + self.capture_size = capture_size + + def capture(self): + # self.cap = cv2.VideoCapture(self.cam_index, cv2.CAP_DSHOW) #windows + self.cap = cv2.VideoCapture(self.cam_index) #linux + width, height = self.capture_size + self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, width) + self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, height) + + def update_frame(self) -> bool: + ret, self.curr_color_frame = self.cap.read() + return ret + + def color_frame(self) -> typing.Union[np.ndarray, None]: + return self.curr_color_frame + + def release(self): + self.cap.release() + + +if __name__ == "__main__": + cam = UVCCamera(0) + cam.capture() + while True: + if not cam.update_frame(): + continue + + frame = cam.color_frame() + if frame is None: + time.sleep(0.01) + continue + + print(frame.shape) + window_name = "preview" + cv2.imshow(window_name, frame) + if cv2.waitKey(1) == ord("q"): + break diff --git a/mycobot_280/mycobot_280arduino/config/EyesInHand_matrix.json b/mycobot_280/mycobot_280arduino/config/EyesInHand_matrix.json new file mode 100644 index 0000000..5db3d2b --- /dev/null +++ b/mycobot_280/mycobot_280arduino/config/EyesInHand_matrix.json @@ -0,0 +1 @@ +[[0.7978586214708513, 0.59877174890628, -0.06995722161683607, -42.33432105525292], [0.6010697464630348, -0.7990332868561791, 0.016154453957772947, -35.01038302335407], [0.04622531807890511, 0.05493813982585287, 0.9974191800647204, 39.342478235028345], [0.0, 0.0, 0.0, 1.0]] \ No newline at end of file diff --git a/mycobot_280/mycobot_280arduino/config/camera_params.npz b/mycobot_280/mycobot_280arduino/config/camera_params.npz new file mode 100644 index 0000000..70b136d Binary files /dev/null and b/mycobot_280/mycobot_280arduino/config/camera_params.npz differ diff --git a/mycobot_280/mycobot_280arduino/config/mycobot_with_marker.rviz b/mycobot_280/mycobot_280arduino/config/mycobot_with_marker.rviz index 8b0dc7f..00873e6 100644 --- a/mycobot_280/mycobot_280arduino/config/mycobot_with_marker.rviz +++ b/mycobot_280/mycobot_280arduino/config/mycobot_with_marker.rviz @@ -126,6 +126,7 @@ Visualization Manager: Value: true joint6_flange: Value: true + Marker Alpha: 1 Marker Scale: 0.300000012 Name: TF Show Arrows: true @@ -145,7 +146,7 @@ Visualization Manager: Value: true - Class: rviz/Marker Enabled: true - Marker Topic: /visualization_marker + Marker Topic: cube Name: Marker Namespaces: basic_cube: true diff --git a/mycobot_280/mycobot_280arduino/launch/detect_marker_with_topic.launch b/mycobot_280/mycobot_280arduino/launch/detect_marker_with_topic.launch new file mode 100644 index 0000000..ed87ec8 --- /dev/null +++ b/mycobot_280/mycobot_280arduino/launch/detect_marker_with_topic.launch @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mycobot_280/mycobot_280arduino/launch/open_camera.launch b/mycobot_280/mycobot_280arduino/launch/open_camera.launch new file mode 100644 index 0000000..b7da730 --- /dev/null +++ b/mycobot_280/mycobot_280arduino/launch/open_camera.launch @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/mycobot_280/mycobot_280arduino/scripts/detect_stag.py b/mycobot_280/mycobot_280arduino/scripts/detect_stag.py new file mode 100644 index 0000000..4a7cb0a --- /dev/null +++ b/mycobot_280/mycobot_280arduino/scripts/detect_stag.py @@ -0,0 +1,445 @@ +#!/usr/bin/env python + +import rospy +from sensor_msgs.msg import Image +from cv_bridge import CvBridge, CvBridgeError +import cv2 +import numpy as np +from numpy.typing import NDArray, ArrayLike +import stag +import os +import json +import time +import threading +from mycobot_communication.msg import MycobotAngles, MycobotSetAngles, MycobotCoords, MycobotSetCoords, MycobotSetEndType, MycobotSetFreshMode, MycobotSetToolReference, MycobotSetVisionMode +from visualization_msgs.msg import Marker + +np.set_printoptions(suppress=True, formatter={'float_kind': '{:.2f}'.format}) + +class STAGRecognizer: + def __init__(self): + rospy.init_node('stag_recognizer', anonymous=True) + self.bridge = CvBridge() + self.tool_len = 20 + + self.marker_size = 32 + self.origin_mycbot_horizontal = [-90, -35.85, -52.91, 88.59, 0, 0.0] + + self.EyesInHand_matrix = None + # 订阅摄像头话题 + self.image_sub = rospy.Subscriber("/usb_cam/image_raw", Image, self.image_callback) + + # 获取config文件目录并设置相机参数文件路径 + file_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + params_file_path = os.path.join(file_dir, "config","camera_params.npz") + print(params_file_path) + matrix_file_path = os.path.join(file_dir, "config","EyesInHand_matrix.json") + self.load_matrix(filename=matrix_file_path) + print(matrix_file_path) + # 加载相机参数 + try: + camera_params = np.load(params_file_path) + self.mtx, self.dist = camera_params["mtx"], camera_params["dist"] + except FileNotFoundError: + rospy.logerr(f"Camera parameters file not found: {params_file_path}") + raise + + self.current_frame = None + + # 创建发布者,发布机械臂坐标和角度 + self.coords_pub = rospy.Publisher('mycobot/coords_goal', MycobotSetCoords, queue_size=5) + self.angles_pub = rospy.Publisher('mycobot/angles_goal', MycobotSetAngles, queue_size=5) + self.fresh_mode_pub = rospy.Publisher('mycobot/fresh_mode_status', MycobotSetFreshMode, queue_size=5) + self.end_type_pub = rospy.Publisher('mycobot/end_type_status', MycobotSetEndType, queue_size=5) + self.tool_reference_pub = rospy.Publisher('mycobot/tool_reference_goal', MycobotSetToolReference, queue_size=5) + self.vision_mode_pub = rospy.Publisher('mycobot/vision_mode_status', MycobotSetVisionMode, queue_size=5) + + # 创建订阅者,订阅机械臂的真实坐标和角度 + rospy.Subscriber('mycobot/coords_real', MycobotCoords, self.coords_callback) + + self.current_coords = None + self.current_angles = None + self.lock = threading.Lock() + self.set_end_type(0) + + # init a node and a publisher + # rospy.init_node("marker", anonymous=True) + self.pub = rospy.Publisher('cube', Marker, queue_size=1) + # init a Marker + self.marker = Marker() + self.marker.header.frame_id = "joint1" + self.marker.ns = "cube" + self.marker.type = self.marker.CUBE + self.marker.action = self.marker.ADD + self.marker.scale.x = 0.04 + self.marker.scale.y = 0.04 + self.marker.scale.z = 0.04 + self.marker.color.a = 1.0 + self.marker.color.g = 1.0 + self.marker.color.r = 1.0 + + # marker position initial + self.marker.pose.position.x = 0 + self.marker.pose.position.y = 0 + self.marker.pose.position.z = 0.03 + self.marker.pose.orientation.x = 0 + self.marker.pose.orientation.y = 0 + self.marker.pose.orientation.z = 0 + self.marker.pose.orientation.w = 1.0 + + def load_matrix(self, filename="EyesInHand_matrix.json"): + # Load the EyesInHand_matrix from a JSON file, if it exists + try: + with open(filename, 'r') as f: + self.EyesInHand_matrix = np.array(json.load(f)) + except FileNotFoundError: + print("Matrix file not found. EyesInHand_matrix will be initialized later.") + + # publish marker + def pub_marker(self, x, y, z=0.03): + self.marker.header.stamp = rospy.Time.now() + self.marker.pose.position.x = x + self.marker.pose.position.y = y + self.marker.pose.position.z = z + self.marker.color.g = 0 + self.pub.publish(self.marker) + + def coords_callback(self, data): + # 获取机械臂当前的坐标,并保留小数点后两位 + with self.lock: + self.current_coords = [round(data.x, 2), round(data.y, 2), round(data.z, 2), + round(data.rx, 2), round(data.ry, 2), round(data.rz, 2)] + self.coords_updated = True + # rospy.loginfo(f"Current coords11111: {self.current_coords}") + + def send_angles(self, angles, speed): + msg = MycobotSetAngles() + msg.joint_1, msg.joint_2, msg.joint_3, msg.joint_4, msg.joint_5, msg.joint_6 = angles + msg.speed = speed + self.angles_pub.publish(msg) + + def send_coords(self, coords, speed, model): + # 创建 MycobotSetCoords 消息对象 + msg = MycobotSetCoords() + + # coords 是一个包含 [x, y, z, rx, ry, rz] 的列表 + if len(coords) != 6: + raise ValueError("coords must be a list of 6 elements") + msg.x, msg.y, msg.z, msg.rx, msg.ry, msg.rz = coords + msg.speed = speed + msg.model = model + + # 发布消息 + self.coords_pub.publish(msg) + + + def get_coords(self): + with self.lock: + if self.coords_updated: + self.coords_updated = False + return self.current_coords.copy() + return [] + + def set_fresh_mode(self, mode): + msg = MycobotSetFreshMode() + msg.Status = mode + self.fresh_mode_pub.publish(msg) + + def set_vision_mode(self, mode): + msg = MycobotSetVisionMode() + msg.Status = mode + self.vision_mode_pub.publish(msg) + + def set_end_type(self, end_type): + msg = MycobotSetEndType() + msg.Status = end_type + self.end_type_pub.publish(msg) + + def set_tool_reference(self, coords): + msg = MycobotSetToolReference() + msg.x, msg.y, msg.z, msg.rx, msg.ry, msg.rz = coords + self.tool_reference_pub.publish(msg) + + + def solve_marker_pnp(self, corners: NDArray, marker_size: int, mtx: NDArray, dist: NDArray): + """ + This will estimate the rvec and tvec for each of the marker corners detected by: + corners, ids, rejectedImgPoints = detector.detectMarkers(image) + corners - is an array of detected corners for each detected marker in the image + marker_size - is the size of the detected markers + mtx - is the camera matrix + distortion - is the camera distortion matrix + RETURN list of rvecs, tvecs, and trash (so that it corresponds to the old estimatePoseSingleMarkers()) + """ + marker_points = np.array( + [ + [-marker_size / 2, marker_size / 2, 0], + [marker_size / 2, marker_size / 2, 0], + [marker_size / 2, -marker_size / 2, 0], + [-marker_size / 2, -marker_size / 2, 0], + ], + dtype=np.float32, + ) + rvecs = [] + tvecs = [] + for corner in corners: + retval, rvec, tvec = cv2.solvePnP( + marker_points, + corner, + mtx, + dist, + flags=cv2.SOLVEPNP_IPPE_SQUARE, + ) + if retval: + rvecs.append(rvec) + tvecs.append(tvec) + + rvecs = np.array(rvecs) # type: ignore + tvecs = np.array(tvecs) # type: ignore + (rvecs - tvecs).any() # type: ignore + return rvecs, tvecs + + def image_callback(self, data): + try: + # 将 ROS 图像消息转换为 OpenCV 格式 + cv_image = self.bridge.imgmsg_to_cv2(data, "bgr8") + self.current_frame = cv_image + except CvBridgeError as e: + rospy.logerr(e) + return + + # 应用相机校正 + # frame_undistorted = cv2.undistort(cv_image, self.mtx, self.dist, None, self.mtx) + + # 检测 STAG 标记 + corners, ids, rejected_corners = stag.detectMarkers(cv_image, 11) + # 绘制检测到的标记及其ID + stag.drawDetectedMarkers(cv_image, corners, ids) + # 绘制被拒绝的候选区域,颜色设为红色 + stag.drawDetectedMarkers(cv_image, rejected_corners, border_color=(255, 0, 0)) + + # cv2.imshow("STAG Detection", cv_image) + # cv2.waitKey(1) + + def calc_markers_base_position(self, corners, ids): + """获取物体坐标(相机系) + + Args: + corners (_type_): _description_ + ids (_type_): _description_ + + Returns: + _type_: _description_ + """ + if len(corners) == 0: + return [] + # 通过二维码角点获取物体旋转向量和平移向量 + rvecs, tvecs = self.solve_marker_pnp(corners, self.marker_size, self.mtx, self.dist) + for i, tvec, rvec in zip(ids, tvecs, rvecs): + tvec = tvec.squeeze().tolist() + rvec = rvec.squeeze().tolist() + rotvector = np.array([[rvec[0], rvec[1], rvec[2]]]) + # 将旋转向量转为旋转矩阵 + Rotation = cv2.Rodrigues(rotvector)[0] + # 将旋转矩阵转为欧拉角 + Euler = self.CvtRotationMatrixToEulerAngle(Rotation) + # 物体坐标(相机系) + target_coords = np.array([tvec[0], tvec[1], tvec[2], Euler[0], Euler[1], Euler[2]]) + return target_coords + + def CvtRotationMatrixToEulerAngle(self, pdtRotationMatrix): + """将旋转矩阵转为欧拉角 + + Args: + pdtRotationMatrix (_type_): _description_ + + Returns: + _type_: _description_ + """ + pdtEulerAngle = np.zeros(3) + pdtEulerAngle[2] = np.arctan2(pdtRotationMatrix[1, 0], pdtRotationMatrix[0, 0]) + fCosRoll = np.cos(pdtEulerAngle[2]) + fSinRoll = np.sin(pdtEulerAngle[2]) + pdtEulerAngle[1] = np.arctan2(-pdtRotationMatrix[2, 0], + (fCosRoll * pdtRotationMatrix[0, 0]) + (fSinRoll * pdtRotationMatrix[1, 0])) + pdtEulerAngle[0] = np.arctan2((fSinRoll * pdtRotationMatrix[0, 2]) - (fCosRoll * pdtRotationMatrix[1, 2]), + (-fSinRoll * pdtRotationMatrix[0, 1]) + (fCosRoll * pdtRotationMatrix[1, 1])) + return pdtEulerAngle + + def CvtEulerAngleToRotationMatrix(self, ptrEulerAngle): + """将欧拉角转为旋转矩阵 + + Args: + ptrEulerAngle (_type_): _description_ + + Returns: + _type_: _description_ + """ + ptrSinAngle = np.sin(ptrEulerAngle) + ptrCosAngle = np.cos(ptrEulerAngle) + ptrRotationMatrix = np.zeros((3, 3)) + ptrRotationMatrix[0, 0] = ptrCosAngle[2] * ptrCosAngle[1] + ptrRotationMatrix[0, 1] = ptrCosAngle[2] * ptrSinAngle[1] * ptrSinAngle[0] - ptrSinAngle[2] * ptrCosAngle[0] + ptrRotationMatrix[0, 2] = ptrCosAngle[2] * ptrSinAngle[1] * ptrCosAngle[0] + ptrSinAngle[2] * ptrSinAngle[0] + ptrRotationMatrix[1, 0] = ptrSinAngle[2] * ptrCosAngle[1] + ptrRotationMatrix[1, 1] = ptrSinAngle[2] * ptrSinAngle[1] * ptrSinAngle[0] + ptrCosAngle[2] * ptrCosAngle[0] + ptrRotationMatrix[1, 2] = ptrSinAngle[2] * ptrSinAngle[1] * ptrCosAngle[0] - ptrCosAngle[2] * ptrSinAngle[0] + ptrRotationMatrix[2, 0] = -ptrSinAngle[1] + ptrRotationMatrix[2, 1] = ptrCosAngle[1] * ptrSinAngle[0] + ptrRotationMatrix[2, 2] = ptrCosAngle[1] * ptrCosAngle[0] + return ptrRotationMatrix + + def Transformation_matrix(self,coord): + """坐标转换为齐次变换矩阵 + + Args: + coord (_type_): (x,y,z,rx,ry,rz) + + Returns: + _type_: _description_ + """ + position_robot = coord[:3] + pose_robot = coord[3:] + # 将欧拉角转为旋转矩阵 + RBT = self.CvtEulerAngleToRotationMatrix(pose_robot) + PBT = np.array([[position_robot[0]], + [position_robot[1]], + [position_robot[2]]]) + temp = np.concatenate((RBT, PBT), axis=1) + array_1x4 = np.array([[0, 0, 0, 1]]) + # 将两个数组按行拼接起来 + matrix = np.concatenate((temp, array_1x4), axis=0) + return matrix + + def Eyes_in_hand(self, coord, marker_positions, Matrix_TC): + # 相机坐标 + Position_Camera = np.transpose(marker_positions[:3]) + # 机械臂坐标矩阵 + Matrix_BT = self.Transformation_matrix(coord) + # 物体坐标(相机系) + Position_Camera = np.append(Position_Camera, 1) + + # 物体坐标(基坐标系) + Position_B = Matrix_BT @ Matrix_TC @ Position_Camera + return Position_B + + def waitl(self, ml): + """等待机械臂运行结束 + + Args: + ml (_type_): _description_ + """ + time.sleep(0.2) + while ml.is_moving(): + time.sleep(0.03) + + def stag_identify(self): + """读取Camera坐标(单次) + + Returns: + _type_: _description_ + """ + try: + if self.current_frame is None: + rospy.logwarn("No image received yet") + return [] + # 获取当前帧 + frame = self.current_frame + # 获取画面中二维码的角度和id + (corners, ids, rejected_corners) = stag.detectMarkers(frame, 11) + # 获取物的坐标(相机系) + marker_pos_pack = self.calc_markers_base_position(corners, ids) + if len(marker_pos_pack) == 0 and not rospy.is_shutdown(): + # rospy.logwarn("No markers detected") + marker_pos_pack, ids = self.stag_identify() # 递归调用 + + # print("Camera coords = ", marker_pos_pack) + return marker_pos_pack, ids + except RecursionError: + # rospy.logerr("Recursion depth exceeded in marker detection") + return [0, 0, 0, 0], 0 # 返回默认值 + + def vision_trace(self, mode, ml): + sp = 40 + #水平面抓取 + if mode == 0: + # 移动到观测点 + ml.send_angles(self.origin_mycbot_horizontal, sp) + # 等待机械臂运动结束 + self.waitl(ml) + input("Enter any key to start trace") + + target_coords = self.stag_robot_identify(ml) + print(target_coords) + + time.sleep(1) + # 机械臂移动到二维码前方 + ml.send_coords(target_coords, 30) + # 等待机械臂运动结束 + self.waitl(ml) + + def stag_robot_identify(self): + marker_pos_pack, ids = self.stag_identify() + # 如果返回的是默认值,直接退出函数,不返回任何数据 + # if marker_pos_pack == [0, 0, 0, 0]: + if np.array_equal(marker_pos_pack, [0, 0, 0, 0]): + rospy.logwarn("No markers detected, skipping processing") + return None # 直接返回 None + target_coords = self.get_coords() + while target_coords is None or len(target_coords) != 6: + target_coords = self.get_coords() + # print("Current coords:", target_coords) + cur_coords = np.array(target_coords.copy()) + cur_coords[-3:] *= (np.pi / 180) + fact_bcl = self.Eyes_in_hand(cur_coords, marker_pos_pack, self.EyesInHand_matrix) + + for i in range(3): + target_coords[i] = fact_bcl[i] + return target_coords, ids + + def coord_limit(self, coords): + min_coord = [-350, -350, 300] + max_coord = [350, 350, 500] + for i in range(3): + if(coords[i] < min_coord[i]): + coords[i] = min_coord[i] + + if(coords[i] > max_coord[i]): + coords[i] = max_coord[i] + + def vision_trace_loop(self): + self.set_fresh_mode(1) + time.sleep(1) + # self.set_vision_mode(0) + # 移动到观测点 + self.send_angles(self.origin_mycbot_horizontal, 50) + time.sleep(3) + origin = self.get_coords() + while origin is None: + origin = self.get_coords() + rate = rospy.Rate(30) + while not rospy.is_shutdown(): + _ ,ids = self.stag_identify() + if ids[0] == 0: + target_coords,_ = self.stag_robot_identify() + # 如果没有返回目标坐标,跳过本次循环 + self.coord_limit(target_coords) + rospy.loginfo('Target Coords: %s', target_coords) + for i in range(3): + target_coords[i+3] = origin[i+3] + self.pub_marker(target_coords[0]/1000.0, target_coords[1]/1000.0, target_coords[2]/1000.0) + self.send_coords(target_coords, 30, 0) # 机械臂移动到二维码前方 + rate.sleep() + elif ids[0] == 1: + self.send_angles(self.origin_mycbot_horizontal, 50) + + +if __name__ == '__main__': + try: + sr = STAGRecognizer() + sr.vision_trace_loop() + rospy.spin() + except KeyboardInterrupt: + rospy.loginfo("Shutting down...") + sr.set_vision_mode(2) + cv2.destroyAllWindows() diff --git a/mycobot_communication/CMakeLists.txt b/mycobot_communication/CMakeLists.txt index aa0f4e9..d352f37 100644 --- a/mycobot_communication/CMakeLists.txt +++ b/mycobot_communication/CMakeLists.txt @@ -59,7 +59,10 @@ add_message_files(FILES MycobotSetEndType.msg MycobotSetFreshMode.msg MycobotSetToolReference.msg + MycobotSetVisionMode.msg + MycobotGetGripperValue.msg + ) ## Generate services in the 'srv' folder @@ -73,6 +76,7 @@ add_service_files(FILES SetEndType.srv SetFreshMode.srv SetToolReference.srv + SetVisionMode.srv ) ## Generate added messages and services diff --git a/mycobot_communication/msg/MycobotSetVisionMode.msg b/mycobot_communication/msg/MycobotSetVisionMode.msg new file mode 100644 index 0000000..ae923e4 --- /dev/null +++ b/mycobot_communication/msg/MycobotSetVisionMode.msg @@ -0,0 +1 @@ +uint8 Status \ No newline at end of file diff --git a/mycobot_communication/scripts/mycobot_topics.py b/mycobot_communication/scripts/mycobot_topics.py index 35c4055..3ddc3c2 100755 --- a/mycobot_communication/scripts/mycobot_topics.py +++ b/mycobot_communication/scripts/mycobot_topics.py @@ -5,6 +5,7 @@ import os import sys import signal import threading +import traceback import rospy @@ -18,6 +19,7 @@ from mycobot_communication.msg import ( MycobotSetEndType, MycobotSetFreshMode, MycobotSetToolReference, + MycobotSetVisionMode, MycobotGetGripperValue, ) from std_msgs.msg import UInt8 @@ -109,6 +111,7 @@ class MycobotTopics(object): rospy.loginfo("%s,%s" % (port, baud)) self.mc = MyCobot280(port, baud) self.lock = threading.Lock() + self.mc.set_vision_mode(1) self.output_robot_message() def start(self): @@ -122,6 +125,8 @@ class MycobotTopics(object): sfm = threading.Thread(target=self.sub_fresh_mode_status) set = threading.Thread(target=self.sub_end_type_status) str = threading.Thread(target=self.sub_set_tool_reference) + svm = threading.Thread(target=self.sub_vision_mode_status) + sgv = threading.Thread(target=self.sub_real_gripper_value) pa.setDaemon(True) @@ -143,6 +148,9 @@ class MycobotTopics(object): set.start() str.setDaemon(True) str.start() + svm.setDaemon(True) + svm.start + sgv.setDaemon(True) sgv.start() @@ -156,6 +164,7 @@ class MycobotTopics(object): sfm.join() set.join() str.join() + svm.join() sgv.join() def pub_real_angles(self): @@ -176,7 +185,10 @@ class MycobotTopics(object): ma.joint_5 = angles[4] ma.joint_6 = angles[5] pub.publish(ma) + else: + rospy.logwarn("None or -1") except Exception as e: + e = traceback.format_exc() rospy.logerr(f"SerialException: {e}") time.sleep(0.25) @@ -199,7 +211,10 @@ class MycobotTopics(object): ma.ry = coords[4] ma.rz = coords[5] pub.publish(ma) + else: + rospy.logwarn("None or -1") except Exception as e: + e = traceback.format_exc() rospy.logerr(f"SerialException: {e}") time.sleep(0.25) @@ -296,6 +311,22 @@ class MycobotTopics(object): "mycobot/fresh_mode_status", MycobotSetFreshMode, callback=callback ) rospy.spin() + + def sub_vision_mode_status(self): + """Subscribe to vision mode Status""" + """订阅运动模式状态""" + def callback(data): + if data.Status==1: + self.mc.set_vision_mode(1) + elif data.Status==2: + self.mc.stop() + else: + self.mc.set_vision_mode(0) + + sub = rospy.Subscriber( + "mycobot/vision_mode_status", MycobotSetVisionMode, callback=callback + ) + rospy.spin() def sub_end_type_status(self): """Subscribe to end type Status""" diff --git a/mycobot_communication/srv/SetVisionMode.srv b/mycobot_communication/srv/SetVisionMode.srv new file mode 100644 index 0000000..42d4322 --- /dev/null +++ b/mycobot_communication/srv/SetVisionMode.srv @@ -0,0 +1,5 @@ +uint8 Status + +--- + +bool Flag \ No newline at end of file