fix: BufferedImage <-> Mat conversion; refacfor: MouseOverText extraction

.gitattributes

@@ -1,3 +1,4 @@
 /gradlew text eol=lf
 *.bat text eol=crlf
 *.jar binary
+* text=auto

build.gradle.kts

@@ -7,10 +7,7 @@ plugins {
 }
 
 group = "com.chromascape"
-version = "0.4.0-SNAPSHOT"
-
-// Customize build directories - put DLLs in build/dist
-layout.buildDirectory.set(file("build"))
+version = "0.4.0"
 
 java {
 	toolchain {
@@ -83,76 +80,3 @@ spotless {
 tasks.named("check") {
 	dependsOn("spotlessApply", "spotlessCheck", "checkstyleMain")
 }
-
-// Windows-only native build configuration
-val isWindows = org.gradle.internal.os.OperatingSystem.current().isWindows
-
-// Copy prebuilt DLLs to build/dist folder
-val copyNativeLibraries by tasks.registering(Copy::class) {
-	group = "native"
-	description = "Copy prebuilt native libraries to build/dist"
-
-	// Only run on Windows
-	onlyIf {
-		isWindows
-	}
-
-	// Check if we have prebuilt libraries
-	onlyIf {
-		val kInputExists = file("third_party/KInput/KInput/KInput/bin/Release/KInput.dll").exists()
-		val kInputCtrlExists = file("third_party/KInput/KInput/KInputCtrl/bin/Release/KInputCtrl.dll").exists()
-
-		kInputExists && kInputCtrlExists
-	}
-
-	doFirst {
-		// Ensure build/dist directory exists
-		file("build/dist").mkdirs()
-	}
-
-	// Copy from prebuilt libraries
-	from("third_party/KInput/KInput/KInput/bin/Release")
-	from("third_party/KInput/KInput/KInputCtrl/bin/Release")
-	into("build/dist")
-
-	include("*.dll")
-}
-
-// Note: Removed copyNativeToResources task as the application loads DLLs directly from build/dist
-// and doesn't use classpath fallback mechanism
-
-// Make build depend on native library copying and quality checks
-tasks.named("processResources") {
-	dependsOn(copyNativeLibraries)
-}
-
-tasks.named("build") {
-	dependsOn(copyNativeLibraries, "check")
-}
-
-tasks.named("jar") {
-	dependsOn(copyNativeLibraries)
-}
-
-// Custom task to clean .chromascape directory
-tasks.register("cleanChromascape") {
-	group = "cleanup"
-	description = "Remove the .chromascape directory"
-
-	doLast {
-		val chromascapeDir = file(".chromascape")
-		if (chromascapeDir.exists()) {
-			delete(chromascapeDir)
-			println("Removed .chromascape directory")
-		} else {
-			println(".chromascape directory does not exist")
-		}
-	}
-}
-
-// Task to clean everything including .chromascape
-tasks.register("cleanAll") {
-	group = "cleanup"
-	description = "Clean build artifacts and .chromascape directory"
-	dependsOn("clean", "cleanChromascape")
-}

src/main/java/com/chromascape/utils/actions/MouseOver.java

@@ -1,85 +1,212 @@
 package com.chromascape.utils.actions;
 
-import static org.bytedeco.opencv.global.opencv_core.bitwise_or;
-import static org.bytedeco.opencv.global.opencv_core.inRange;
-import static org.bytedeco.opencv.global.opencv_imgproc.COLOR_BGR2HSV;
-import static org.bytedeco.opencv.global.opencv_imgproc.cvtColor;
-import static org.opencv.core.CvType.CV_8UC1;
+import static org.bytedeco.opencv.global.opencv_core.CV_8UC1;
+import static org.bytedeco.opencv.global.opencv_core.CV_8UC3;
 
 import com.chromascape.base.BaseScript;
 import com.chromascape.utils.core.screen.colour.ColourObj;
+import com.chromascape.utils.core.screen.topology.ColourContours;
+import com.chromascape.utils.core.screen.topology.TemplateMatching;
 import com.chromascape.utils.core.screen.window.ScreenManager;
 import com.chromascape.utils.domain.ocr.Ocr;
 import java.awt.Rectangle;
 import java.awt.image.BufferedImage;
 import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
-import org.bytedeco.javacv.Java2DFrameUtils;
+import java.util.Set;
+import org.bytedeco.javacpp.indexer.UByteIndexer;
 import org.bytedeco.opencv.opencv_core.Mat;
 import org.bytedeco.opencv.opencv_core.Scalar;
 
 /**
- * An actions utility to provide a high level API for MouseOverText.
+ * An actions utility to provide a high level API for MouseOverText. Allows the user to get the text
+ * of the MouseOverText zone regardless of colour.
  *
- * <p>Uses OpenCV to iterate over a list of colours, and collates the resulting image into one
- * overall mask. This allows the user to get the text of the whole MouseOverText zone regardless of
- * colour.
- *
- * <p>Allows the user to grab the MouseOverText immediately as a string, excluding spaces.
+ * <p>Allows the user to grab the MouseOverText as a string, excluding spaces.
  */
 public class MouseOver {
 
-  /** Colours that exist within the MouseOverText zone. */
-  private static final List<ColourObj> colours =
-      new ArrayList<>(
-          Arrays.asList(
-              new ColourObj("TEXT_CYAN", new Scalar(80, 180, 200, 0), new Scalar(100, 255, 255, 0)),
-              new ColourObj(
-                  "TEXT_OFF_WHITE", new Scalar(0, 0, 190, 0), new Scalar(180, 30, 255, 0)),
-              new ColourObj("TEXT_ORANGE", new Scalar(8, 140, 180, 0), new Scalar(22, 220, 255, 0)),
-              new ColourObj("TEXT_GREEN", new Scalar(50, 190, 100, 0), new Scalar(95, 255, 255, 0)),
-              new ColourObj(
-                  "TEXT_YELLOW", new Scalar(25, 130, 190, 0), new Scalar(35, 255, 255, 0)),
-              new ColourObj("TEXT_RED", new Scalar(0, 190, 190, 0), new Scalar(8, 255, 255, 0))));
+  // Higher -> brighter pixels are considered shadows
+  private static final int SHADOW_THRESHOLD = 120;
+
+  // Lower -> stricter tolerance on similar colours
+  private static final int COLOUR_TOLERANCE = 20;
+
+  private static final ColourObj ORANGE =
+      new ColourObj("Orange", new Scalar(16, 224, 255, 0), new Scalar(16, 224, 255, 0));
+
+  private static final Mat OLIVE = new Mat(1, 1, CV_8UC3, new Scalar(17, 73, 73, 0));
 
   /**
-   * Captures the minimap to extract all possible colours. Layers the captures to create a mask
-   * containing all text regardless of colour. Searches for text based on this.
+   * Extracts text from the MouseOverText zone. Does not include spaces.
    *
-   * @param baseScript Your script instance, typically {@code this}.
-   * @return The string found within the MouseOverText zone (No spaces).
+   * @param baseScript {@code this} the script that the user is running.
+   * @return a String of all text found.
    */
   public static String getText(BaseScript baseScript) {
     // Get image of MouseOverText
     Rectangle zone = baseScript.controller().zones().getMouseOver();
     BufferedImage capture = ScreenManager.captureZone(zone);
+    // BGR image of the zone
+    Mat image = TemplateMatching.bufferedImageToMat(capture);
+    // Replace orange to olive to exclude interface colours near zone
+    Mat orangeMask = ColourContours.extractColours(image, ORANGE);
+    image.setTo(OLIVE, orangeMask);
+    // Release mask
+    orangeMask.release();
+    // Extract possible character colours based on shadow
+    Set<Integer> textColours = extractTextColour(image);
+    Object[] uniqueTextColours = removeDuplicatesWithinRange(textColours);
+    // Set non character pixels to black
+    Mat mask = maskText(uniqueTextColours, image);
+    // For testing
+    // DisplayImage.display(TemplateMatching.matToBufferedImage(mask));
+    return Ocr.extractTextFromMask(mask, "Bold 12", true);
+  }
+
+  /**
+   * Eliminates any colours that can be considered similar enough within the colour threshold.
+   *
+   * @param textColours a list of packed RGB integers.
+   * @return Unique colours that can't be considered similar.
+   */
+  private static Object[] removeDuplicatesWithinRange(Set<Integer> textColours) {
+    List<Integer> palette = new ArrayList<>();
+
+    for (Integer newColour : textColours) {
+      boolean found = false;
+
+      for (Integer existingColour : palette) {
+
+        if (isInThreshold(existingColour, newColour)) {
+          found = true;
+          break;
+        }
+      }
 
-    // Convert the captured BGR image to HSV once here,
-    // so we don't have to do it inside the loop for every single colour
-    Mat hsvMat = new Mat();
-    try (Mat bgrMat = Java2DFrameUtils.toMat(capture)) {
-      cvtColor(bgrMat, hsvMat, COLOR_BGR2HSV);
+      if (!found) {
+        palette.add(newColour);
+      }
     }
+    return palette.toArray();
+  }
+
+  /**
+   * Compares each pixel in an image against a palette of unique colours that represent text colours
+   * in an image. Creates a mask with white pixels where text should be and the background black.
+   *
+   * @param colours a list of unique colours not within the colour threshold of each other.
+   * @param image the image containing text to mask.
+   * @return a CV_8UC1 mask with white pixels where text should be and the rest black.
+   */
+  private static Mat maskText(Object[] colours, Mat image) {
+    Mat mask = new Mat(image.rows(), image.cols(), CV_8UC1, new Scalar(0));
+    UByteIndexer img = image.createIndexer();
+    UByteIndexer out = mask.createIndexer();
+
+    for (int y = 0; y < image.rows(); y++) {
+      for (int x = 0; x < image.cols(); x++) {
+        int b = img.get(y, x, 0) & 0xFF;
+        int g = img.get(y, x, 1) & 0xFF;
+        int r = img.get(y, x, 2) & 0xFF;
+        int packed = (r << 16) | (g << 8) | b;
 
-    // Accumulate all colour matches into a single binary mask
-    try (Mat combinedMask =
-            new Mat(capture.getHeight(), capture.getWidth(), CV_8UC1, new Scalar(0));
-        Mat tempMask = new Mat()) { // Reusable mask for the loop using try with resources
-
-      for (ColourObj c : colours) {
-        // In memory thresholding using the pre-converted HSV Mat
-        try (Mat min = new Mat(c.hsvMin());
-            Mat max = new Mat(c.hsvMax())) {
-          inRange(hsvMat, min, max, tempMask);
-          bitwise_or(combinedMask, tempMask, combinedMask);
+        boolean isWhite = false;
+
+        for (Object colour : colours) {
+          Integer colourInt = (Integer) colour;
+
+          if (isInThreshold(colourInt, packed)) {
+            isWhite = true;
+            break;
+          }
         }
+
+        out.put(y, x, isWhite ? 255 : 0);
       }
+    }
+
+    img.release();
+    out.release();
+    return mask;
+  }
+
+  /**
+   * Compares the Euclidean distance between the RGB values of two pixels against a static tolerance
+   * value to gauge if they are similar enough.
+   *
+   * @param colour1 one of the colours to compare.
+   * @param colour2 one of the colours to compare.
+   * @return whether the colours are similar enough to consider part of the same text colour.
+   */
+  private static boolean isInThreshold(int colour1, int colour2) {
+    // Unpack first colour
+    int r = (colour1 >> 16) & 0xFF;
+    int g = (colour1 >> 8) & 0xFF;
+    int b = colour1 & 0xFF;
+    // Unpack second
+    int r2 = (colour2 >> 16) & 0xFF;
+    int g2 = (colour2 >> 8) & 0xFF;
+    int b2 = colour2 & 0xFF;
+    // Calculate difference
+    int dr = r - r2;
+    int dg = g - g2;
+    int db = b - b2;
+    // Calculate distance^2
+    int dist2 = dr * dr + dg * dg + db * db;
+    return dist2 <= COLOUR_TOLERANCE * COLOUR_TOLERANCE;
+  }
 
-      // Cleanup
-      hsvMat.release();
+  private static boolean isShadow(int r, int g, int b) {
+    return ((r + g + b) / 3 < SHADOW_THRESHOLD)
+        && (r < SHADOW_THRESHOLD * 2)
+        && (g < SHADOW_THRESHOLD * 2)
+        && (b < SHADOW_THRESHOLD * 2);
+  }
 
-      return Ocr.extractTextFromMask(combinedMask, "Bold 12", true);
+  /**
+   * Iterates through an image's pixels, determines whether a pixel is a shadow of text based on
+   * brightness, and then saves the pixel top-left of it. Returns a set of unique pixels that are
+   * considered to be part of text.
+   *
+   * @param image the input image which may contain text.
+   * @return a unique set of packed RGB integers representing pixels belonging to text.
+   */
+  private static Set<Integer> extractTextColour(Mat image) {
+    int width = image.cols();
+    int height = image.rows();
+
+    UByteIndexer indexer = image.createIndexer();
+
+    Set<Integer> textColours = new HashSet<>();
+
+    for (int y = 1; y < height; y++) {
+      for (int x = 1; x < width; x++) {
+
+        int blue = indexer.get(y, x, 0) & 0xFF;
+        int green = indexer.get(y, x, 1) & 0xFF;
+        int red = indexer.get(y, x, 2) & 0xFF;
+
+        if (isShadow(red, green, blue)) {
+          int matchBlue = indexer.get(y - 1, x - 1, 0) & 0xFF;
+          int matchGreen = indexer.get(y - 1, x - 1, 1) & 0xFF;
+          int matchRed = indexer.get(y - 1, x - 1, 2) & 0xFF;
+
+          if (isShadow(matchRed, matchGreen, matchBlue)) {
+            continue;
+          }
+
+          int r = matchRed & 0xFF;
+          int g = matchGreen & 0xFF;
+          int b = matchBlue & 0xFF;
+
+          int packed = (r << 16) | (g << 8) | b;
+
+          textColours.add(packed);
+        }
+      }
     }
+    return textColours;
   }
 }

src/main/java/com/chromascape/utils/core/screen/DisplayImage.java

@@ -6,7 +6,6 @@
 import javax.swing.JFrame;
 import javax.swing.JLabel;
 import javax.swing.WindowConstants;
-import org.bytedeco.javacv.Java2DFrameUtils;
 
 /**
  * Utility class for temporarily displaying {@link BufferedImage} instances in a Swing window for
@@ -26,7 +25,7 @@ public class DisplayImage {
    * <p>This method is primarily intended for testing and debugging purposes during development.
    *
    * @param image The image to display. If the source is an OpenCV {@code Mat}, convert it first
-   *     using {@link Java2DFrameUtils#toBufferedImage(org.bytedeco.opencv.opencv_core.Mat)}.
+   *     using {@code TemplateMatching.matToBufferedImage(Mat)}.
    */
   public static void display(BufferedImage image) {
     if (frame == null) {

src/main/java/com/chromascape/utils/core/screen/topology/ColourContours.java

@@ -14,7 +14,6 @@
 import java.awt.image.BufferedImage;
 import java.util.ArrayList;
 import java.util.List;
-import org.bytedeco.javacv.Java2DFrameUtils;
 import org.bytedeco.opencv.opencv_core.*;
 
 /**
@@ -96,7 +95,7 @@ public static ChromaObj getChromaObjClosestToCentre(List<ChromaObj> chromaObjs)
    */
   public static Mat extractColours(BufferedImage image, ColourObj colourObj) {
     // Convert BufferedImage to Mat explicitly
-    try (Mat hsvImage = Java2DFrameUtils.toMat(image)) {
+    try (Mat hsvImage = TemplateMatching.bufferedImageToMat(image)) {
       return extractColours(hsvImage, colourObj);
     }
   }