Merge pull request JdeRobot#3529 from JdeRobot/add-image-viewer-machine-vision

Add image viewer in machine vision exercise

Author: javizqh

database/exercises/db.sql

@@ -289,10 +289,11 @@ COPY public.exercises_tools (id, exercise_id, tool_id) FROM stdin;
 72	25	console
 73	25	simulator
 74	25	rviz
-75	26	console
-76	26	simulator
-77	26	web_gui
-78	15	video
+75	25	web_gui
+76	26	console
+77	26	simulator
+78	26	web_gui
+79	15	video
 \.
 
 --

exercises/machine_vision/frontend/WebGUI.tsx

@@ -0,0 +1,37 @@
+import React, { useState } from "react";
+import { useExercise } from "Contexts/ExerciseContext";
+import WebGUIContainer, {
+  connectApplication,
+} from "Components/exercise/WebGUIContainer";
+import WebGUIImage from "Components/exercise/WebGUIImage";
+
+const WebGUI = () => {
+  const exerciseContext = useExercise();
+  const manager = exerciseContext.manager;
+
+  const [image, setImage] = useState<string | undefined>(undefined);
+
+  const updateCallback = (updateData: unknown) => {
+    const data = updateData as any;
+
+    if (data.update?.image_right) {
+      const img = JSON.parse(data.update.image_right);
+
+      if (img.image_right !== "") {
+        setImage(`data:image/jpeg;base64,${img.image_right}`);
+      }
+    }
+  };
+
+  const stateCallback = () => {};
+
+  connectApplication(manager, updateCallback, stateCallback);
+
+  return (
+    <WebGUIContainer>
+      <WebGUIImage style={{ width: "100%" }} src={image} />
+    </WebGUIContainer>
+  );
+};
+
+export default WebGUI;

exercises/machine_vision/frontend/eslint.config.mts

@@ -0,0 +1,29 @@
+import js from "@eslint/js";
+import globals from "globals";
+import tseslint from "typescript-eslint";
+import pluginReact from "eslint-plugin-react";
+import { defineConfig } from "eslint/config";
+
+export default defineConfig([
+  {
+    files: ["**/*.{js,mjs,cjs,ts,mts,cts,jsx,tsx}"],
+    plugins: { js },
+    extends: ["js/recommended"],
+    languageOptions: {
+      globals: globals.browser,
+      parserOptions: {
+        project: './tsconfig.json',
+        tsconfigRootDir: import.meta.dirname
+      },
+    },
+  },
+  tseslint.configs.recommended,
+  pluginReact.configs.flat.recommended,
+  {
+    settings: {
+      react: {
+        version: "detect",
+      },
+    },
+  },
+]);

exercises/machine_vision/frontend/tsconfig.json

@@ -0,0 +1,60 @@
+{
+  "compilerOptions": {
+    "target": "es5",
+    "lib": [
+      "dom",
+      "dom.iterable",
+      "esnext"
+    ],
+    "types": [
+      "../../../react_frontend/svg.d.ts",
+      "../../../react_frontend/png.d.ts",
+      "../../../react_frontend/jpg.d.ts",
+      "../../../react_frontend/zip.d.ts"
+    ],
+    "paths": {
+      "Assets/*": [ "../../../react_frontend/src/assets/*"],
+      "Components/*": [ "../../../react_frontend/src/components/*"],
+      "Utils/*": [ "../../../react_frontend/src/utils/*"],
+      "Contexts/*": [ "../../../react_frontend/src/contexts/*"],
+      "Helpers/*": [ "../../../react_frontend/src/helpers/*"],
+      "Icons/*": [ "../../../react_frontend/src/icons/*"],
+      "Styles/*": [ "../../../react_frontend/src/styles/*"],
+      "Types/*": [ "../../../react_frontend/src/types/*"],
+      "Constants/*": [ "../../../react_frontend/src/constants/*"],
+      "Api": [ "../../../react_frontend/src/api/index.ts"],
+      "Routes": [ "../../../react_frontend/src/routes/index.ts"],
+      "*": [ "../../../react_frontend/node_modules/*"],
+    },
+    "baseUrl": "./",
+    "rootDirs": [
+      "./",
+      "../../../react_frontend/src"
+    ],
+    "typeRoots": ["../../../react_frontend/node_modules/@types"],
+    "declaration": true,
+    "allowJs": true,
+    "skipLibCheck": true,
+    "esModuleInterop": true,
+    "allowSyntheticDefaultImports": true,
+    "strict": true,
+    "noImplicitAny": true,
+    "forceConsistentCasingInFileNames": true,
+    "noFallthroughCasesInSwitch": true,
+    "module": "esnext",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "noEmit": true,
+    "jsx": "react-jsx"
+  },
+  "include": [
+    "./",
+    "eslint.config.mts",
+    "../../../react_frontend/src",
+    "../../../react_frontend/png.d.ts",
+    "../../../react_frontend/jpg.d.ts",
+    "../../../react_frontend/svg.d.ts",
+    "../../../react_frontend/zip.d.ts"
+  ]
+}

exercises/machine_vision/python_template/HAL.py

@@ -42,6 +42,7 @@
 from geometry_msgs.msg import Point, Pose, PoseStamped, Quaternion
 from pcl_filter_msgs.msg import ColorFilter, ShapeFilter
 from tf_transformations import euler_from_quaternion, quaternion_from_euler
+from hal_interfaces.general.camera import CameraNode
 
 # Build PATH and import Python classes from IFRA package:
 PATH = os.path.join(get_package_share_directory("ros2srrc_execution"), "python")
@@ -58,11 +59,30 @@
 from ros2srrc_data.msg import Action, Joint, Joints, Xyz, Ypr, Robpose
 
 # Global variables
-rclpy.init(args=None)
+if not rclpy.ok():
+    rclpy.init(args=None)
 UR5 = RBT()
 _home_joints = [0.0, -90.0, 0.0, 0.0, -90.0, 0.0]
 _perception_manager = None
 
+# Camera node
+camera_node = CameraNode("/hand_camera/image_raw")
+
+# Spin nodes so that subscription callbacks load topic data
+executor = MultiThreadedExecutor()
+executor.add_node(camera_node)
+
+def __auto_spin():
+    while rclpy.ok():
+        try:
+            executor.spin_once(timeout_sec=0)
+        except Exception:
+            pass
+        time.sleep(1/90.0)
+
+executor_thread = threading.Thread(target=__auto_spin, daemon=True)
+executor_thread.start()
+
 #################################### UTILITY CLASSES ###################################################
 
 
@@ -1039,6 +1059,14 @@ def get_target_position(target_name):
     return perception_mgr.get_target_position(target_name)
 
 
+def getImage():
+    image = camera_node.getImage()
+
+    while image is None:
+        image = camera_node.getImage()
+
+    return image.data
+
 #################################### MAIN FUNCTION ###################################################
 
 

exercises/machine_vision/python_template/WebGUI.py

@@ -17,7 +17,6 @@ def __init__(self, host="ws://127.0.0.1:2303", freq=30.0):
         # Execution control vars
         self.out_period = 1.0 / freq
         self.right_image = None
-        self.left_image = None
         self.image_lock = threading.Lock()
         self.ack = True
         self.ack_frontend = True
@@ -27,7 +26,7 @@ def __init__(self, host="ws://127.0.0.1:2303", freq=30.0):
         self.world_name = "empty"
 
         self.host = host
-        self.msg = {"image_right": "", "image_left": ""}
+        self.msg = {"image_right": ""}
 
         self.ideal_cycle = 80
         self.real_time_factor = 0
@@ -48,7 +47,7 @@ def gui_out_thread(self):
             with self.ack_lock:
                 with self.image_lock:
                     if self.ack:
-                        if np.any(self.left_image) or np.any(self.right_image):
+                        if np.any(self.right_image):
                             self.update_gui()
                             self.ack = False
 
@@ -60,14 +59,6 @@ def gui_out_thread(self):
     # Prepares and send image to the websocket server
     def update_gui(self):
 
-        if np.any(self.left_image):
-            _, encoded_left_image = cv2.imencode(".JPEG", self.left_image)
-            b64_left = base64.b64encode(encoded_left_image).decode("utf-8")
-            shape_left = self.left_image.shape
-        else:
-            b64_left = None
-            shape_left = 0
-
         if np.any(self.right_image):
             _, encoded_right_image = cv2.imencode(".JPEG", self.right_image)
             b64_right = base64.b64encode(encoded_right_image).decode("utf-8")
@@ -76,24 +67,16 @@ def update_gui(self):
             b64_right = None
             shape_right = 0
 
-        payload_left = {
-            "image_left": b64_left,
-            "shape_left": shape_left,
-        }
         payload_right = {
             "image_right": b64_right,
             "shape_right": shape_right,
         }
-        self.msg["image_left"] = json.dumps(payload_left)
+
         self.msg["image_right"] = json.dumps(payload_right)
         message = json.dumps(self.msg)
         self.send_to_client(message)
 
     # Functions to set the next image to be sent
-    def setLeftImage(self, image):
-        with self.image_lock:
-            self.left_image = image
-
     def setRightImage(self, image):
         with self.image_lock:
             self.right_image = image
@@ -108,8 +91,4 @@ def setRightImage(self, image):
 
 # Expose the user functions
 def showImage(image):
-    gui.setRightImage(image)
-
-
-def showLeftImage(image):
-    gui.setLeftImage(image)
+    gui.setRightImage(image)