diff --git a/sketch-to-ui-plugin/build.gradle.kts b/sketch-to-ui-plugin/build.gradle.kts new file mode 100644 index 0000000..7b81438 --- /dev/null +++ b/sketch-to-ui-plugin/build.gradle.kts @@ -0,0 +1,91 @@ +import org.jetbrains.kotlin.gradle.dsl.JvmTarget + +plugins { + id("com.android.application") + id("org.jetbrains.kotlin.android") + id("com.itsaky.androidide.plugins.build") +} + +pluginBuilder { + pluginName = "sketch-to-ui" +} + +android { + namespace = "com.appdevforall.sketchtoui.plugin" + compileSdk = 35 + + defaultConfig { + applicationId = "com.appdevforall.sketchtoui.plugin" + minSdk = 28 + targetSdk = 35 + versionCode = 1 + versionName = "1.0.0" + } + + buildFeatures { + viewBinding = true + } + + buildTypes { + release { + isMinifyEnabled = false + proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro") + } + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } + + packaging { + resources { + excludes += setOf( + "META-INF/versions/9/OSGI-INF/MANIFEST.MF", + "META-INF/DEPENDENCIES", + "META-INF/LICENSE", + "META-INF/LICENSE.txt", + "META-INF/NOTICE", + "META-INF/NOTICE.txt" + ) + } + } +} + +kotlin { + compilerOptions { + jvmTarget.set(JvmTarget.JVM_17) + } +} + +dependencies { + compileOnly(files("../libs/plugin-api.jar")) + + implementation("androidx.appcompat:appcompat:1.6.1") + implementation("com.google.android.material:material:1.10.0") + implementation("androidx.activity:activity-ktx:1.9.3") + implementation("androidx.core:core-ktx:1.13.1") + implementation("androidx.exifinterface:exifinterface:1.3.7") + implementation("androidx.fragment:fragment-ktx:1.8.8") + implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.8.7") + implementation("androidx.lifecycle:lifecycle-viewmodel-ktx:2.8.7") + implementation("org.jetbrains.kotlin:kotlin-stdlib:2.3.0") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.9.0") + implementation("io.insert-koin:koin-android:4.0.0") + implementation("com.google.code.gson:gson:2.11.0") + implementation("me.xdrop:fuzzywuzzy:1.4.0") + implementation("com.google.ai.edge.litert:litert:1.0.1") + implementation("com.google.ai.edge.litert:litert-support:1.0.1") + implementation("com.google.ai.edge.litert:litert-gpu:1.0.1") + implementation("com.google.mlkit:text-recognition:16.0.1") + implementation("com.google.android.gms:play-services-mlkit-text-recognition-common:19.1.0") + implementation("com.google.mlkit:vision-common:17.3.0") + implementation("com.google.mlkit:common:18.11.0") + + testImplementation("junit:junit:4.13.2") +} + +tasks.wrapper { + gradleVersion = "8.10.2" + distributionType = Wrapper.DistributionType.BIN +} diff --git a/sketch-to-ui-plugin/gradle.properties b/sketch-to-ui-plugin/gradle.properties new file mode 100644 index 0000000..1139ec7 --- /dev/null +++ b/sketch-to-ui-plugin/gradle.properties @@ -0,0 +1,11 @@ +# Project-wide Gradle settings +android.useAndroidX=true +android.enableJetifier=true + +# Kotlin +kotlin.code.style=official + +# Performance optimizations +org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8 +org.gradle.parallel=true +org.gradle.caching=true \ No newline at end of file diff --git a/sketch-to-ui-plugin/gradle/wrapper/gradle-wrapper.jar b/sketch-to-ui-plugin/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..d64cd49 Binary files /dev/null and b/sketch-to-ui-plugin/gradle/wrapper/gradle-wrapper.jar differ diff --git a/sketch-to-ui-plugin/gradle/wrapper/gradle-wrapper.properties b/sketch-to-ui-plugin/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..42f0071 --- /dev/null +++ b/sketch-to-ui-plugin/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists \ No newline at end of file diff --git a/sketch-to-ui-plugin/gradlew b/sketch-to-ui-plugin/gradlew new file mode 100644 index 0000000..1aa94a4 --- /dev/null +++ b/sketch-to-ui-plugin/gradlew @@ -0,0 +1,249 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/sketch-to-ui-plugin/gradlew.bat b/sketch-to-ui-plugin/gradlew.bat new file mode 100644 index 0000000..93e3f59 --- /dev/null +++ b/sketch-to-ui-plugin/gradlew.bat @@ -0,0 +1,92 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/sketch-to-ui-plugin/proguard-rules.pro b/sketch-to-ui-plugin/proguard-rules.pro new file mode 100644 index 0000000..923af53 --- /dev/null +++ b/sketch-to-ui-plugin/proguard-rules.pro @@ -0,0 +1,22 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle.kts. + +# Keep plugin main class +-keep class com.example.sampleplugin.SamplePlugin { *; } + +# Keep all plugin fragments +-keep class com.example.sampleplugin.fragments.** { *; } + +# Keep plugin interfaces +-keep interface com.itsaky.androidide.plugins.** { *; } +-keep class com.itsaky.androidide.plugins.** { *; } + +# Keep Android components +-keepclassmembers class * extends androidx.fragment.app.Fragment { + public (...); +} + +# Keep reflection-based APIs +-keepattributes Signature +-keepattributes *Annotation* \ No newline at end of file diff --git a/sketch-to-ui-plugin/settings.gradle.kts b/sketch-to-ui-plugin/settings.gradle.kts new file mode 100644 index 0000000..9894cc7 --- /dev/null +++ b/sketch-to-ui-plugin/settings.gradle.kts @@ -0,0 +1,30 @@ +pluginManagement { + repositories { + google() + mavenCentral() + gradlePluginPortal() + } +} + +buildscript { + repositories { + google() + mavenCentral() + } + dependencies { + classpath(files("../libs/plugin-api.jar")) + classpath(files("../libs/gradle-plugin.jar")) + classpath("com.android.tools.build:gradle:8.11.0") + classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:2.3.0") + } +} + +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { + google() + mavenCentral() + } +} + +rootProject.name = "sketch-to-ui-plugin" diff --git a/sketch-to-ui-plugin/src/main/AndroidManifest.xml b/sketch-to-ui-plugin/src/main/AndroidManifest.xml new file mode 100644 index 0000000..1b5fd64 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/AndroidManifest.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/sketch-to-ui-plugin/src/main/assets/best_float32.tflite b/sketch-to-ui-plugin/src/main/assets/best_float32.tflite new file mode 100644 index 0000000..6593e43 Binary files /dev/null and b/sketch-to-ui-plugin/src/main/assets/best_float32.tflite differ diff --git a/sketch-to-ui-plugin/src/main/assets/icon_day.png b/sketch-to-ui-plugin/src/main/assets/icon_day.png new file mode 100644 index 0000000..6695551 Binary files /dev/null and b/sketch-to-ui-plugin/src/main/assets/icon_day.png differ diff --git a/sketch-to-ui-plugin/src/main/assets/icon_night.png b/sketch-to-ui-plugin/src/main/assets/icon_night.png new file mode 100644 index 0000000..ef3cfb9 Binary files /dev/null and b/sketch-to-ui-plugin/src/main/assets/icon_night.png differ diff --git a/sketch-to-ui-plugin/src/main/assets/labels.txt b/sketch-to-ui-plugin/src/main/assets/labels.txt new file mode 100644 index 0000000..04ea3c3 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/assets/labels.txt @@ -0,0 +1,13 @@ +generic_box +dropdown_symbol +image_placeholder +button +checkbox_checked +checkbox_unchecked +radio_button_unchecked +radio_button_checked +slider +switch_off +switch_on +widget_tag +margin_metadata \ No newline at end of file diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/data/repository/DrawableImportHelper.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/data/repository/DrawableImportHelper.kt new file mode 100644 index 0000000..6a32981 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/data/repository/DrawableImportHelper.kt @@ -0,0 +1,174 @@ +package org.appdevforall.codeonthego.computervision.data.repository + +import android.content.ContentResolver +import android.net.Uri +import android.provider.OpenableColumns +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.io.File +import java.util.Locale + +/** + * Helper class responsible for importing and managing image files within an Android project's + * resource directory (e.g., res/drawable). + * + * @property contentResolver The ContentResolver used to read data from content URIs. + */ +class DrawableImportHelper( + private val contentResolver: ContentResolver +) { + + /** + * Imports an image from a given URI into the 'res/drawable' directory associated with + * the provided layout file. + * + * @param sourceUri The URI of the image to import. + * @param layoutFilePath The absolute path to the current layout XML file. Used to locate the 'res' directory. + * @param fallbackName A name to use if the original file name cannot be resolved. + * @return A [Result] containing an [ImportedDrawable] if successful, or an exception on failure. + */ + suspend fun importDrawable( + sourceUri: Uri, + layoutFilePath: String?, + fallbackName: String + ): Result = withContext(Dispatchers.IO) { + runCatching { + requireNotNull(layoutFilePath) { "Layout file path is not available." } + + val drawableDir = getOrCreateDrawableDirectory(layoutFilePath) + val extension = resolveSupportedExtension(sourceUri, fallbackName) + val baseName = sanitizeResourceName(resolveDisplayName(sourceUri) ?: fallbackName) + val destinationFile = resolveAvailableFile(drawableDir, baseName, extension) + + copyImageToDestination(sourceUri, destinationFile) + + ImportedDrawable( + resourceName = destinationFile.nameWithoutExtension, + drawableReference = "@drawable/${destinationFile.nameWithoutExtension}", + file = destinationFile + ) + } + } + + /** + * Deletes an imported drawable file from the filesystem. + * + * @param layoutFilePath The absolute path to the current layout XML file. Used to locate the 'res' directory. + * @param resourceName The sanitized name of the resource to delete (without extension). + * @return A [Result] indicating success (true if deleted, false if file did not exist) or failure. + */ + suspend fun deleteDrawable( + layoutFilePath: String?, + resourceName: String + ): Result = withContext(Dispatchers.IO) { + runCatching { + requireNotNull(layoutFilePath) { "Layout file path is not available." } + + val drawableDir = resolveDrawableDir(File(layoutFilePath)) + if (!drawableDir.exists()) return@runCatching false + + val targetFile = findFileByResourceName(drawableDir, resourceName) + + targetFile?.delete() ?: false + } + } + + private fun getOrCreateDrawableDirectory(layoutFilePath: String): File { + val layoutFile = File(layoutFilePath) + val drawableDir = resolveDrawableDir(layoutFile) + check(drawableDir.exists() || drawableDir.mkdirs()) { + "Could not create drawable directory: ${drawableDir.absolutePath}" + } + return drawableDir + } + + private fun copyImageToDestination(sourceUri: Uri, destinationFile: File) { + contentResolver.openInputStream(sourceUri)?.use { input -> + destinationFile.outputStream().use(input::copyTo) + } ?: error("Could not open selected image.") + } + + private fun findFileByResourceName(directory: File, resourceName: String): File? { + return directory.listFiles()?.firstOrNull { + it.nameWithoutExtension == resourceName + } + } + + private fun resolveDrawableDir(layoutFile: File): File { + val resDir = generateSequence(layoutFile.parentFile) { it.parentFile } + .firstOrNull { it.name == "res" } + ?: throw IllegalStateException("Could not resolve res directory from: ${layoutFile.absolutePath}") + + return File(resDir, "drawable") + } + + private fun resolveDisplayName(uri: Uri): String? { + return contentResolver.query(uri, arrayOf(OpenableColumns.DISPLAY_NAME), null, null, null) + ?.use { cursor -> + val index = cursor.getColumnIndex(OpenableColumns.DISPLAY_NAME) + if (index >= 0 && cursor.moveToFirst()) cursor.getString(index) else null + } + } + + private fun resolveSupportedExtension(uri: Uri, fallbackName: String): String { + val mimeType = contentResolver.getType(uri)?.lowercase(Locale.US) + var extension = when (mimeType) { + "image/png" -> "png" + "image/jpeg", "image/jpg" -> "jpg" + "image/webp" -> "webp" + else -> null + } + + if (extension == null) { + val nameToUse = resolveDisplayName(uri) ?: fallbackName + extension = nameToUse + .substringAfterLast('.', missingDelimiterValue = "") + .lowercase(Locale.US) + .takeIf { it.isNotBlank() } + } + + return when (extension) { + "png", "jpg", "jpeg", "webp" -> extension + else -> throw IllegalArgumentException("Unsupported image format. Use PNG, JPG, JPEG, or WEBP.") + } + } + + private fun sanitizeResourceName(rawName: String): String { + val nameWithoutExtension = rawName.substringBeforeLast('.') + val normalized = nameWithoutExtension + .lowercase(Locale.US) + .replace(Regex("[^a-z0-9_]"), "_") + .replace(Regex("_+"), "_") + .trim('_') + + val safeName = normalized.ifBlank { "imported_image" } + + return if (safeName.first().isDigit()) { + "img_$safeName" + } else { + safeName + } + } + + private fun resolveAvailableFile( + drawableDir: File, + baseName: String, + extension: String + ): File { + var candidate = File(drawableDir, "$baseName.$extension") + var index = 1 + + while (!candidate.createNewFile()) { + candidate = File(drawableDir, "${baseName}_$index.$extension") + index++ + } + + return candidate + } +} + +data class ImportedDrawable( + val resourceName: String, + val drawableReference: String, + val file: File +) diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/data/repository/VisionRepository.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/data/repository/VisionRepository.kt new file mode 100644 index 0000000..c4713f1 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/data/repository/VisionRepository.kt @@ -0,0 +1,18 @@ +package org.appdevforall.codeonthego.computervision.data.repository + +import android.graphics.Bitmap +import com.google.mlkit.vision.text.Text +import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult + +/** + * Abstract contract for computer vision data sources. + * Handles raw interactions with machine learning models (YOLO, MLKit) + * without leaking domain logic. + */ +interface VisionRepository { + suspend fun initModel(): Result + suspend fun detectWidgets(bitmap: Bitmap): Result> + suspend fun recognizeText(bitmap: Bitmap): Result> + fun isInitialized(): Boolean + fun release() +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/data/repository/VisionRepositoryImpl.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/data/repository/VisionRepositoryImpl.kt new file mode 100644 index 0000000..305e55a --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/data/repository/VisionRepositoryImpl.kt @@ -0,0 +1,39 @@ +package org.appdevforall.codeonthego.computervision.data.repository + +import android.content.res.AssetManager +import android.graphics.Bitmap +import com.google.mlkit.vision.text.Text +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import org.appdevforall.codeonthego.computervision.data.source.OcrSource +import org.appdevforall.codeonthego.computervision.data.source.YoloModelSource +import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult + +class VisionRepositoryImpl( + private val assetManager: AssetManager, + private val yoloModelSource: YoloModelSource, + private val ocrSource: OcrSource +) : VisionRepository { + + override suspend fun initModel(): Result = withContext(Dispatchers.IO) { + runCatching { + yoloModelSource.initialize(assetManager) + } + } + + override suspend fun detectWidgets(bitmap: Bitmap): Result> = + withContext(Dispatchers.Default) { + runCatching { yoloModelSource.runInference(bitmap) } + } + + override suspend fun recognizeText(bitmap: Bitmap): Result> = + withContext(Dispatchers.Default) { + ocrSource.recognizeText(bitmap) + } + + override fun isInitialized(): Boolean = yoloModelSource.isInitialized() + + override fun release() { + yoloModelSource.release() + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/data/source/MlKitInitializer.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/data/source/MlKitInitializer.kt new file mode 100644 index 0000000..bf46f3a --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/data/source/MlKitInitializer.kt @@ -0,0 +1,39 @@ +package org.appdevforall.codeonthego.computervision.data.source + +import android.content.Context +import android.content.ContextWrapper +import com.google.firebase.components.ComponentRegistrar +import com.google.mlkit.common.internal.CommonComponentRegistrar +import com.google.mlkit.common.sdkinternal.MlKitContext +import com.google.mlkit.vision.common.internal.VisionCommonRegistrar +import com.google.mlkit.vision.text.internal.TextRegistrar +import java.util.concurrent.atomic.AtomicBoolean + +object MlKitInitializer { + private val initialized = AtomicBoolean(false) + + fun initialize(context: Context): Context { + val mlKitContext = MlKitContextWrapper(context) + if (initialized.get()) { + return mlKitContext + } + + MlKitContext.initializeIfNeeded(mlKitContext, registrars()) + initialized.set(true) + return mlKitContext + } + + private fun registrars(): List { + return listOf( + TextRegistrar(), + VisionCommonRegistrar(), + CommonComponentRegistrar() + ) + } + + private class MlKitContextWrapper(context: Context) : ContextWrapper(context) { + override fun getApplicationContext(): Context { + return this + } + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/data/source/OcrSource.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/data/source/OcrSource.kt new file mode 100644 index 0000000..ef7f8a8 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/data/source/OcrSource.kt @@ -0,0 +1,43 @@ +package org.appdevforall.codeonthego.computervision.data.source + +import android.content.Context +import android.graphics.Bitmap +import com.google.mlkit.vision.common.InputImage +import com.google.mlkit.vision.text.Text +import com.google.mlkit.vision.text.TextRecognition +import com.google.mlkit.vision.text.latin.TextRecognizerOptions +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlin.coroutines.resume + +class OcrSource(context: Context) { + + private val safeContext: Context = MlKitInitializer.initialize(context) + + private val textRecognizer by lazy { + TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS) + } + + suspend fun recognizeText(bitmap: Bitmap): Result> = + suspendCancellableCoroutine { continuation -> + runCatching { + val safeBitmap = bitmap.safeCopyForMlKit() + InputImage.fromBitmap(safeBitmap, 0) + }.onSuccess { inputImage -> + textRecognizer.process(inputImage) + .addOnSuccessListener { visionText -> + continuation.resume(Result.success(visionText.textBlocks)) + } + .addOnFailureListener { exception -> + continuation.resume(Result.failure(exception)) + } + }.onFailure { exception -> + continuation.resume(Result.failure(exception)) + } + } + + private fun Bitmap.safeCopyForMlKit(): Bitmap { + check(!isRecycled) { "Bitmap is recycled" } + check(width > 0 && height > 0) { "Bitmap has invalid size ${width}x${height}" } + return if (config == Bitmap.Config.ARGB_8888) this else copy(Bitmap.Config.ARGB_8888, false) + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/data/source/YoloModelSource.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/data/source/YoloModelSource.kt new file mode 100644 index 0000000..6c305eb --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/data/source/YoloModelSource.kt @@ -0,0 +1,156 @@ +package org.appdevforall.codeonthego.computervision.data.source + +import android.content.res.AssetManager +import android.graphics.Bitmap +import android.graphics.RectF +import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult +import org.tensorflow.lite.DataType +import org.tensorflow.lite.Interpreter +import org.tensorflow.lite.support.common.ops.CastOp +import org.tensorflow.lite.support.common.ops.NormalizeOp +import org.tensorflow.lite.support.image.ImageProcessor +import org.tensorflow.lite.support.image.TensorImage +import org.tensorflow.lite.support.image.ops.ResizeOp +import org.tensorflow.lite.support.tensorbuffer.TensorBuffer +import java.io.FileInputStream +import java.io.IOException +import java.nio.MappedByteBuffer +import java.nio.channels.FileChannel + +class YoloModelSource { + + private var interpreter: Interpreter? = null + private var labels: List = emptyList() + + companion object { + private const val MODEL_INPUT_WIDTH = 640 + private const val MODEL_INPUT_HEIGHT = 640 + private const val CONFIDENCE_THRESHOLD = 0.2f + private const val NMS_THRESHOLD = 0.45f + } + + @Throws(IOException::class) + fun initialize(assetManager: AssetManager, modelPath: String = "best_float32.tflite", labelsPath: String = "labels.txt") { + interpreter = Interpreter(loadModelFile(assetManager, modelPath)) + labels = assetManager.open(labelsPath).bufferedReader() + .useLines { lines -> lines.map { it.trim() }.toList() } + } + + fun isInitialized(): Boolean = interpreter != null && labels.isNotEmpty() + + fun runInference(bitmap: Bitmap): List { + val interp = interpreter ?: throw IllegalStateException("Model not initialized") + + val imageProcessor = ImageProcessor.Builder() + .add(ResizeOp(MODEL_INPUT_HEIGHT, MODEL_INPUT_WIDTH, ResizeOp.ResizeMethod.BILINEAR)) + .add(NormalizeOp(0.0f, 255.0f)) + .add(CastOp(DataType.FLOAT32)) + .build() + + val tensorImage = imageProcessor.process(TensorImage.fromBitmap(bitmap)) + val outputShape = interp.getOutputTensor(0).shape() + val outputBuffer = TensorBuffer.createFixedSize(outputShape, DataType.FLOAT32) + + interp.run(tensorImage.buffer, outputBuffer.buffer.rewind()) + + return processYoloOutput(outputBuffer, bitmap.width, bitmap.height) + } + + fun release() { + interpreter?.close() + interpreter = null + } + + private fun processYoloOutput(buffer: TensorBuffer, imageWidth: Int, imageHeight: Int): List { + val shape = buffer.shape + val numProperties = shape[1] + val numPredictions = shape[2] + val numClasses = numProperties - 4 + val floatArray = buffer.floatArray + + val transposedArray = FloatArray(shape[0] * numPredictions * numProperties) + for (i in 0 until numPredictions) { + for (j in 0 until numProperties) { + transposedArray[i * numProperties + j] = floatArray[j * numPredictions + i] + } + } + + val allDetections = mutableListOf() + for (i in 0 until numPredictions) { + val offset = i * numProperties + var maxClassScore = 0f + var classId = -1 + for (j in 0 until numClasses) { + val classScore = transposedArray[offset + 4 + j] + if (classScore > maxClassScore) { + maxClassScore = classScore + classId = j + } + } + if (maxClassScore > CONFIDENCE_THRESHOLD) { + val x = transposedArray[offset + 0] + val y = transposedArray[offset + 1] + val w = transposedArray[offset + 2] + val h = transposedArray[offset + 3] + + val left = (x - w / 2) * imageWidth + val top = (y - h / 2) * imageHeight + val right = (x + w / 2) * imageWidth + val bottom = (y + h / 2) * imageHeight + + val label = labels.getOrElse(classId) { "Unknown" } + allDetections.add(DetectionResult(RectF(left, top, right, bottom), label, maxClassScore)) + } + } + + return applyNms(allDetections) + } + + private fun applyNms(detections: List): List { + val finalDetections = mutableListOf() + val groupedByLabel = detections.groupBy { it.label } + + for ((_, group) in groupedByLabel) { + val sortedDetections = group.sortedByDescending { it.score } + val remaining = sortedDetections.toMutableList() + + while (remaining.isNotEmpty()) { + val bestDetection = remaining.first() + finalDetections.add(bestDetection) + remaining.remove(bestDetection) + + val iterator = remaining.iterator() + while (iterator.hasNext()) { + val detection = iterator.next() + if (calculateIoU(bestDetection.boundingBox, detection.boundingBox) > NMS_THRESHOLD) { + iterator.remove() + } + } + } + } + + return finalDetections + } + + private fun calculateIoU(box1: RectF, box2: RectF): Float { + val xA = maxOf(box1.left, box2.left) + val yA = maxOf(box1.top, box2.top) + val xB = minOf(box1.right, box2.right) + val yB = minOf(box1.bottom, box2.bottom) + + val intersectionArea = maxOf(0f, xB - xA) * maxOf(0f, yB - yA) + val box1Area = box1.width() * box1.height() + val box2Area = box2.width() * box2.height() + val unionArea = box1Area + box2Area - intersectionArea + + return if (unionArea == 0f) 0f else intersectionArea / unionArea + } + + @Throws(IOException::class) + private fun loadModelFile(assetManager: AssetManager, modelPath: String): MappedByteBuffer { + val fileDescriptor = assetManager.openFd(modelPath) + return FileInputStream(fileDescriptor.fileDescriptor).use { inputStream -> + inputStream.channel.map(FileChannel.MapMode.READ_ONLY, fileDescriptor.startOffset, fileDescriptor.declaredLength) + } + } +} \ No newline at end of file diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/di/ComputerVisionModule.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/di/ComputerVisionModule.kt new file mode 100644 index 0000000..13798e1 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/di/ComputerVisionModule.kt @@ -0,0 +1,57 @@ +package org.appdevforall.codeonthego.computervision.di + +import android.content.Context +import org.appdevforall.codeonthego.computervision.data.repository.DrawableImportHelper +import org.appdevforall.codeonthego.computervision.data.repository.VisionRepository +import org.appdevforall.codeonthego.computervision.data.repository.VisionRepositoryImpl +import org.appdevforall.codeonthego.computervision.data.source.OcrSource +import org.appdevforall.codeonthego.computervision.data.source.YoloModelSource +import org.appdevforall.codeonthego.computervision.domain.GenericBoxResolver +import org.appdevforall.codeonthego.computervision.domain.RegionOcrProcessor +import org.appdevforall.codeonthego.computervision.domain.usecase.GenerateXmlUC +import org.appdevforall.codeonthego.computervision.domain.usecase.ImportPlaceholderImageUC +import org.appdevforall.codeonthego.computervision.domain.usecase.PrepareImageUC +import org.appdevforall.codeonthego.computervision.domain.usecase.RemovePlaceholderImageUC +import org.appdevforall.codeonthego.computervision.domain.usecase.RunVisionUC +import org.appdevforall.codeonthego.computervision.ui.viewmodel.ComputerVisionViewModel +import org.koin.android.ext.koin.androidContext +import org.koin.core.module.dsl.viewModel +import org.koin.core.scope.Scope +import org.koin.dsl.module + +fun computerVisionModule(contextProvider: Scope.() -> Context) = module { + + single { YoloModelSource() } + single { OcrSource(contextProvider()) } + single { RegionOcrProcessor(ocrSource = get()) } + single { GenericBoxResolver() } + + single { + val context = contextProvider() + VisionRepositoryImpl( + assetManager = context.assets, + yoloModelSource = get(), + ocrSource = get() + ) + } + + single { DrawableImportHelper(contentResolver = contextProvider().contentResolver) } + single { GenerateXmlUC() } + single { ImportPlaceholderImageUC(drawableImportHelper = get()) } + single { PrepareImageUC(contentResolver = contextProvider().contentResolver) } + single { RemovePlaceholderImageUC(drawableImportHelper = get()) } + single { RunVisionUC(repository = get(), boxResolver = get(), regionOcrProcessor = get()) } + + viewModel { (layoutFilePath: String?, layoutFileName: String?) -> + ComputerVisionViewModel( + repository = get(), + prepareImageUC = get(), + runVisionUC = get(), + generateXmlUC = get(), + importPlaceholderImageUC = get(), + removePlaceholderImageUC = get(), + layoutFilePath = layoutFilePath, + layoutFileName = layoutFileName + ) + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/DetectionMerger.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/DetectionMerger.kt new file mode 100644 index 0000000..1878eac --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/DetectionMerger.kt @@ -0,0 +1,62 @@ +package org.appdevforall.codeonthego.computervision.domain + +import android.graphics.RectF +import com.google.mlkit.vision.text.Text +import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult +import org.appdevforall.codeonthego.computervision.utils.OcrTextAssembler.joinElementsWithTolerance + +class DetectionMerger( + private val enrichedComponents: List, + private val remainingYoloDetections: List, + private val fullImageTextBlocks: List +) { + + private val containerLabels = setOf("card", "toolbar") + + fun merge(): List { + val finalDetections = mutableListOf() + val usedTextBlocks = mutableSetOf() + + finalDetections.addAll(enrichedComponents) + finalDetections.addAll(remainingYoloDetections) + + val containers = remainingYoloDetections.filter { it.label in containerLabels } + for (container in containers) { + val candidates = fullImageTextBlocks.filter { it !in usedTextBlocks } + for (textBlock in candidates) { + val textBox = textBlock.boundingBox?.let { RectF(it) } ?: continue + if (container.boundingBox.contains(textBox)) { + finalDetections.add( + DetectionResult( + boundingBox = textBox, + label = "text", + score = 0.99f, + text = textBlock.text.replace("\n", " "), + isYolo = false + ) + ) + usedTextBlocks.add(textBlock) + } + } + } + + val orphanDetections = fullImageTextBlocks + .filter { it !in usedTextBlocks } + .flatMap { it.lines } + .mapNotNull { line -> + line.boundingBox?.let { box -> + DetectionResult( + boundingBox = RectF(box), + label = "text", + score = 0.99f, + text = joinElementsWithTolerance(line), + isYolo = false + ) + } + } + + finalDetections.addAll(orphanDetections) + + return finalDetections + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/DetectionScaler.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/DetectionScaler.kt new file mode 100644 index 0000000..5f08087 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/DetectionScaler.kt @@ -0,0 +1,38 @@ +package org.appdevforall.codeonthego.computervision.domain + +import android.graphics.Rect +import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult +import org.appdevforall.codeonthego.computervision.domain.model.ScaledBox +import kotlin.math.max +import kotlin.math.roundToInt + +/** + * Scales the normalized YOLO coordinates (0.0 to 1.0) to the target dimensions + * in DP (e.g., 360x640) of the Android screen. + */ +object DetectionScaler { + private const val MIN_W_ANY = 8 + private const val MIN_H_ANY = 8 + + fun scale( + detection: DetectionResult, sourceWidth: Int, sourceHeight: Int, targetW: Int, targetH: Int + ): ScaledBox { + if (sourceWidth == 0 || sourceHeight == 0) { + return ScaledBox(detection.label, detection.text, 0, 0, MIN_W_ANY, MIN_H_ANY, MIN_W_ANY / 2, MIN_H_ANY / 2, Rect(0, 0, MIN_W_ANY, MIN_H_ANY)) + } + val rect = detection.boundingBox + val normCx = ((rect.left + rect.right) / 2f) / sourceWidth.toFloat() + val normCy = ((rect.top + rect.bottom) / 2f) / sourceHeight.toFloat() + val normW = (rect.right - rect.left) / sourceWidth.toFloat() + val normH = (rect.bottom - rect.top) / sourceHeight.toFloat() + + val x = max(0, ((normCx - normW / 2f) * targetW).roundToInt()) + val y = max(0, ((normCy - normH / 2f) * targetH).roundToInt()) + val w = max(MIN_W_ANY, (normW * targetW).roundToInt()) + val h = max(MIN_H_ANY, (normH * targetH).roundToInt()) + + return ScaledBox( + detection.label, detection.text, x, y, w, h, x + w / 2, y + h / 2, Rect(x, y, x + w, y + h) + ) + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/GenericBoxResolver.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/GenericBoxResolver.kt new file mode 100644 index 0000000..bcb16e7 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/GenericBoxResolver.kt @@ -0,0 +1,37 @@ +package org.appdevforall.codeonthego.computervision.domain + +import android.graphics.RectF +import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult +import kotlin.math.hypot + +class GenericBoxResolver { + + fun resolve(detections: List): List { + val dropdownSymbols = detections.filter { it.label == "dropdown_symbol" } + + return detections.mapNotNull { det -> + when (det.label) { + "dropdown_symbol" -> null + "generic_box" -> { + val hasSymbolNearby = dropdownSymbols.any { symbol -> + isNearby(det.boundingBox, symbol.boundingBox, 0.8f) + } + det.copy(label = if (hasSymbolNearby) "dropdown" else "text_entry_box") + } + else -> det + } + } + } + + private fun isNearby(box1: RectF, box2: RectF, thresholdFactor: Float = 1.5f): Boolean { + val avgDim1 = (box1.width() + box1.height()) / 2f + val distanceThreshold = thresholdFactor * avgDim1 + + val distance = hypot( + (box1.centerX() - box2.centerX()).toDouble(), + (box1.centerY() - box2.centerY()).toDouble() + ).toFloat() + + return distance < distanceThreshold + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/LayoutTreeBuilder.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/LayoutTreeBuilder.kt new file mode 100644 index 0000000..0877095 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/LayoutTreeBuilder.kt @@ -0,0 +1,110 @@ +package org.appdevforall.codeonthego.computervision.domain + +import org.appdevforall.codeonthego.computervision.domain.model.LayoutItem +import org.appdevforall.codeonthego.computervision.domain.model.ScaledBox +import kotlin.math.abs +import kotlin.math.max + +/** + * Analyzes the spatial distribution of the detected boxes and builds + * a logical visual hierarchy (tree of Layouts, Rows, RadioGroups, etc.) + * based on vertical alignment and overlap. + */ +object LayoutTreeBuilder { + private const val OVERLAP_THRESHOLD = 0.6 + private const val VERTICAL_ALIGN_THRESHOLD = 20 + + fun buildLayoutTree(boxes: List): List { + val rows = groupIntoRows(boxes) + val items = mutableListOf() + val verticalRadioRun = mutableListOf() + val verticalCheckboxRun = mutableListOf() + + fun flushRuns() { + if (verticalRadioRun.isNotEmpty()) { + items.add(LayoutItem.RadioGroup(verticalRadioRun.toList(), "vertical")) + verticalRadioRun.clear() + } + if (verticalCheckboxRun.isNotEmpty()) { + items.add(LayoutItem.CheckboxGroup(verticalCheckboxRun.toList(), "vertical")) + verticalCheckboxRun.clear() + } + } + + rows.forEach { row -> + val isRadioRow = row.all { isRadioButton(it) } + val isCheckboxRow = row.all { isCheckbox(it) } + + if (!isRadioRow && verticalRadioRun.isNotEmpty()) flushRuns() + if (!isCheckboxRow && verticalCheckboxRun.isNotEmpty()) flushRuns() + + when { + isRadioRow && row.size == 1 -> verticalRadioRun.add(row.first()) + isRadioRow -> items.add(LayoutItem.RadioGroup(row, "horizontal")) + isCheckboxRow && row.size == 1 -> verticalCheckboxRun.add(row.first()) + isCheckboxRow -> items.add(LayoutItem.CheckboxGroup(row, "horizontal")) + else -> { + flushRuns() + if (row.size == 1) { + items.add(LayoutItem.SimpleView(row.first())) + } else { + items.add(LayoutItem.HorizontalRow(row)) + } + } + } + } + flushRuns() + + return items + } + + private fun groupIntoRows(boxes: List): List> { + val rows = mutableListOf() + + boxes.sortedWith(compareBy({ it.y }, { it.x })).forEach { box -> + val row = rows.firstOrNull { it.accepts(box) } + if (row == null) { + rows.add(LayoutRow(box)) + } else { + row.add(box) + } + } + + return rows.sortedBy { it.top }.map { it.boxes.sortedBy(ScaledBox::x) } + } + + private fun isRadioButton(box: ScaledBox): Boolean = + box.label == "radio_button_unchecked" || box.label == "radio_button_checked" + + private fun isCheckbox(box: ScaledBox): Boolean = + box.label == "checkbox_unchecked" || box.label == "checkbox_checked" + + private class LayoutRow(initialBox: ScaledBox) { + private val _boxes = mutableListOf(initialBox) + val boxes: List get() = _boxes + + var top: Int = initialBox.y + private set + var bottom: Int = initialBox.y + initialBox.h + private set + + val height: Int get() = bottom - top + val centerY: Int get() = top + height / 2 + + fun add(box: ScaledBox) { + _boxes.add(box) + top = minOf(top, box.y) + bottom = maxOf(bottom, box.y + box.h) + } + + fun accepts(box: ScaledBox): Boolean { + val verticalOverlap = minOf(box.y + box.h, bottom) - maxOf(box.y, top) + val minHeight = minOf(box.h, height).coerceAtLeast(1) + val overlapRatio = verticalOverlap.toFloat() / minHeight.toFloat() + val centerDelta = abs(box.centerY - centerY) + val centerThreshold = max(VERTICAL_ALIGN_THRESHOLD, minHeight / 2) + + return overlapRatio >= OVERLAP_THRESHOLD || centerDelta <= centerThreshold + } + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/MarginAnnotationParser.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/MarginAnnotationParser.kt new file mode 100644 index 0000000..02c33c8 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/MarginAnnotationParser.kt @@ -0,0 +1,231 @@ +package org.appdevforall.codeonthego.computervision.domain + +import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult +import org.appdevforall.codeonthego.computervision.utils.MetadataDetector +import kotlin.math.abs + +object MarginAnnotationParser { + + /** + * Extracts canvas UI elements and parses margin annotations, linking them together. + * * @param detections The full list of detections from YOLO and OCR. + * @param imageWidth The width of the source image in pixels. + * @param leftGuidePct The percentage (0.0 to 1.0) defining the left margin boundary. + * @param rightGuidePct The percentage (0.0 to 1.0) defining the right margin boundary. + * @return A Pair containing the valid canvas UI detections and a mapped dictionary of [Widget Tag -> Annotation Text]. + */ + fun parse( + detections: List, + imageWidth: Int, + leftGuidePct: Float, + rightGuidePct: Float + ): Pair, Map> { + val sanitizedDetections = detections.filterNot { MetadataDetector.isMetadataLabel(it.label) } + + val distribution = distributeDetections(sanitizedDetections, imageWidth, leftGuidePct, rightGuidePct) + val canvasTags = extractCanvasTags(distribution.canvas) + + val annotationMap = parseMarginsGlobally( + leftMargin = distribution.leftMargin, + rightMargin = distribution.rightMargin, + canvasTags = canvasTags + ) + + return Pair(distribution.canvas, annotationMap) + } + + /** + * Distributes raw detections into three zones: left margin, right margin, and the main canvas. + * Metadata detections invading the canvas are forcefully pushed back to the margins. + */ + private fun distributeDetections( + detections: List, + imageWidth: Int, + leftGuidePct: Float, + rightGuidePct: Float + ): DetectionDistribution { + val leftMarginPx = imageWidth * leftGuidePct + val rightMarginPx = imageWidth * rightGuidePct + + val canvas = mutableListOf() + val leftMargin = mutableListOf() + val rightMargin = mutableListOf() + + for (detection in detections) { + val isMetadata = MetadataDetector.isCanvasMetadata(detection.text) + val centerX = centerX(detection) + + when { + isMetadata && centerX < (imageWidth / 2f) -> leftMargin.add(detection) + isMetadata && centerX >= (imageWidth / 2f) -> rightMargin.add(detection) + centerX > leftMarginPx && centerX < rightMarginPx -> canvas.add(detection) + centerX <= leftMarginPx -> leftMargin.add(detection) + else -> rightMargin.add(detection) + } + } + + return DetectionDistribution(canvas, leftMargin, rightMargin) + } + + /** + * Identifies and extracts valid widget tags from the canvas detections. + */ + private fun extractCanvasTags(canvasDetections: List): List> { + return canvasDetections.mapNotNull { det -> + WidgetTagParser.extractTag(det.text)?.let { (tag, _) -> tag to det } + } + } + + /** + * Processes both margins simultaneously to prevent cross-margin collisions. + * Gathers all explicit annotations first, then resolves all implicit blocks + * against a shared pool of remaining tags. + */ + private fun parseMarginsGlobally( + leftMargin: List, + rightMargin: List, + canvasTags: List> + ): Map { + val leftBlocks = extractBlocks(leftMargin.sortedBy { it.boundingBox.top }) + val rightBlocks = extractBlocks(rightMargin.sortedBy { it.boundingBox.top }) + + val globalExplicitAnnotations = mergeAnnotations( + leftBlocks.explicitAnnotations, + rightBlocks.explicitAnnotations + ) + + val allImplicitBlocks = leftBlocks.implicitBlocks + rightBlocks.implicitBlocks + + val resolvedImplicitAnnotations = resolveImplicitBlocks( + implicitBlocks = allImplicitBlocks, + canvasTags = canvasTags, + existingAnnotations = globalExplicitAnnotations + ) + + return mergeAnnotations(globalExplicitAnnotations, resolvedImplicitAnnotations) + } + + /** + * Merges multiple annotation maps. If a tag exists in multiple maps, + * their values are combined separated by " | ". + */ + private fun mergeAnnotations(vararg maps: Map): MutableMap { + return maps.flatMap { it.toList() } + .groupBy({ it.first }, { it.second }) + .mapValues { (_, values) -> values.joinToString(" | ") } + .toMutableMap() + } + + /** + * Reads vertically through margin detections and groups them into text blocks. + * Blocks starting with an explicit tag become explicit annotations, while untagged blocks are stored as implicit. + * Side-based prefix heuristics are intentionally not applied here because OCR can place + * a valid explicit tag on the opposite margin from its detected canvas tag. + */ + private fun extractBlocks( + sortedDetections: List + ): GroupedBlocks { + val blocks = GroupedBlocks() + var currentTag: String? = null + var currentText = StringBuilder() + var blockStartY = 0f + + fun saveCurrentBlock() { + if (currentTag != null) { + blocks.explicitAnnotations[currentTag!!] = currentText.toString().trim() + } else if (currentText.isNotBlank()) { + blocks.implicitBlocks.add(ParsedBlock(currentText.toString().trim(), blockStartY)) + } + } + + for (det in sortedDetections) { + val text = det.text.trim().trimStart('|', ':', ';', '.', ',', '_') + val extraction = WidgetTagParser.extractTag(text) + + val isExplicitTag = extraction != null + + if (isExplicitTag) { + saveCurrentBlock() + + currentTag = extraction.first + currentText = StringBuilder() + blockStartY = centerY(det) + + val trailing = extraction.second?.trim() + if (!trailing.isNullOrBlank() && WidgetTagParser.normalizeTagText(trailing) != currentTag) { + currentText.append(trailing).append(" ") + } + } else { + if (currentText.isEmpty()) blockStartY = centerY(det) + currentText.append(text).append(" ") + } + } + saveCurrentBlock() + + return blocks + } + + /** + * Resolves implicit (untagged) text blocks by associating them with the nearest vertical canvas tag + * of the most appropriate prefix. + */ + private fun resolveImplicitBlocks( + implicitBlocks: List, + canvasTags: List>, + existingAnnotations: Map + ): Map { + val resolvedAnnotations = mutableMapOf() + + val canvasTagsByPrefix = canvasTags + .groupBy { (tag, _) -> tag.substringBefore('-') } + .mapValues { (_, tags) -> tags.sortedBy { (_, det) -> centerY(det) } } + + val unresolvedTagsByPrefix = canvasTagsByPrefix + .mapValues { (_, tags) -> + tags.map { it.first } + .filter { tag -> tag !in existingAnnotations } + .sortedBy { tag -> WidgetTagParser.extractOrdinal(tag) ?: Int.MAX_VALUE } + .toMutableList() + }.toMutableMap() + + for (block in implicitBlocks.sortedBy { it.centerY }) { + val closestPrefix = unresolvedTagsByPrefix + .filterValues { it.isNotEmpty() } + .minByOrNull { (prefix, remainingTags) -> + val nearestTagY = canvasTagsByPrefix[prefix] + ?.firstOrNull { (tag, _) -> tag == remainingTags.firstOrNull() } + ?.second?.let { centerY(it) } ?: Float.MAX_VALUE + abs(nearestTagY - block.centerY) + }?.key ?: continue + + val assignedTag = unresolvedTagsByPrefix[closestPrefix]?.removeFirstOrNull() ?: continue + resolvedAnnotations[assignedTag] = block.annotationText + } + + return resolvedAnnotations + } + + private fun centerX(detection: DetectionResult): Float { + return (detection.boundingBox.left + detection.boundingBox.right) / 2f + } + + private fun centerY(detection: DetectionResult): Float { + return (detection.boundingBox.top + detection.boundingBox.bottom) / 2f + } + + private data class DetectionDistribution( + val canvas: List, + val leftMargin: List, + val rightMargin: List + ) + + private data class GroupedBlocks( + val explicitAnnotations: MutableMap = mutableMapOf(), + val implicitBlocks: MutableList = mutableListOf() + ) + + private data class ParsedBlock( + val annotationText: String, + val centerY: Float + ) +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/RegionOcrProcessor.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/RegionOcrProcessor.kt new file mode 100644 index 0000000..72653ba --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/RegionOcrProcessor.kt @@ -0,0 +1,136 @@ +package org.appdevforall.codeonthego.computervision.domain + +import android.graphics.Bitmap +import android.graphics.RectF +import com.google.mlkit.vision.text.Text +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import org.appdevforall.codeonthego.computervision.data.source.OcrSource +import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult +import org.appdevforall.codeonthego.computervision.utils.BitmapUtils +import org.appdevforall.codeonthego.computervision.utils.OcrTextAssembler + +class RegionOcrProcessor( + private val ocrSource: OcrSource, + private val componentPadding: Int = 10 +) { + + private val interactiveLabels = setOf( + "button", + "switch_on", + "switch_off", + "text_entry_box", + "dropdown", + "radio_button_checked", + "radio_button_unchecked", + "slider", + "image_placeholder", + "widget_tag" + ) + + data class RegionOcrResult( + val enrichedDetections: List, + val remainingDetections: List, + val marginDetections: List, + val fullImageTextBlocks: List + ) + + suspend fun process( + originalBitmap: Bitmap, + yoloDetections: List, + leftGuidePct: Float, + rightGuidePct: Float + ): RegionOcrResult = coroutineScope { + val interactive = yoloDetections.filter { it.label in interactiveLabels } + val remaining = yoloDetections.filter { it.label !in interactiveLabels } + + val widgetOcrDeferred = async { runWidgetOcr(originalBitmap, interactive) } + val marginOcrDeferred = async { runMarginOcr(originalBitmap, leftGuidePct, rightGuidePct) } + val fullImageOcrDeferred = async { runFullImageOcr(originalBitmap) } + + RegionOcrResult( + enrichedDetections = widgetOcrDeferred.await(), + remainingDetections = remaining, + marginDetections = marginOcrDeferred.await(), + fullImageTextBlocks = fullImageOcrDeferred.await() + ) + } + + private suspend fun runWidgetOcr( + bitmap: Bitmap, + components: List + ): List = coroutineScope { + components.map { component -> + async { + var crop: Bitmap? = null + var preprocessed: Bitmap? = null + try { + crop = BitmapUtils.cropRegion(bitmap, component.boundingBox, componentPadding) + preprocessed = BitmapUtils.preprocessForOcr(crop) + val textBlocks = ocrSource.recognizeText(preprocessed).getOrNull() + val text = textBlocks?.let { OcrTextAssembler.extractTextWithTolerance(it) } ?: "" + component.copy(text = text) + } finally { + preprocessed?.recycle() + if (crop != null && crop !== bitmap) crop.recycle() + } + } + }.awaitAll() + } + + private suspend fun runMarginOcr( + bitmap: Bitmap, + leftGuidePct: Float, + rightGuidePct: Float + ): List { + val width = bitmap.width.toFloat() + val height = bitmap.height.toFloat() + val results = mutableListOf() + + val leftRect = RectF(0f, 0f, width * leftGuidePct, height) + results.addAll(ocrCroppedRegion(bitmap, leftRect, 0f)) + + val rightOffsetX = width * rightGuidePct + val rightRect = RectF(rightOffsetX, 0f, width, height) + results.addAll(ocrCroppedRegion(bitmap, rightRect, rightOffsetX)) + + return results + } + + private suspend fun ocrCroppedRegion( + bitmap: Bitmap, + rect: RectF, + offsetX: Float + ): List { + val crop = BitmapUtils.cropRegion(bitmap, rect) + if (crop === bitmap) return emptyList() + return try { + val textBlocks = ocrSource.recognizeText(crop).getOrNull() ?: emptyList() + textBlocks.flatMap { block -> + block.lines.mapNotNull { line -> + line.boundingBox?.let { box -> + DetectionResult( + boundingBox = RectF( + box.left + offsetX, + box.top + rect.top, + box.right + offsetX, + box.bottom + rect.top + ), + label = "text", + score = 0.99f, + text = OcrTextAssembler.joinElementsWithTolerance(line), + isYolo = false + ) + } + } + } + } finally { + crop.recycle() + } + } + + private suspend fun runFullImageOcr(bitmap: Bitmap): List { + return ocrSource.recognizeText(bitmap).getOrNull() ?: emptyList() + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/TextAssociator.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/TextAssociator.kt new file mode 100644 index 0000000..b1eac0d --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/TextAssociator.kt @@ -0,0 +1,106 @@ +package org.appdevforall.codeonthego.computervision.domain + +import android.graphics.Rect +import org.appdevforall.codeonthego.computervision.domain.model.ScaledBox +import org.appdevforall.codeonthego.computervision.utils.TextCleaner.cleanTextPreservingLeadingO +import org.appdevforall.codeonthego.computervision.utils.TextCleaner.cleanTextStrippingLeadingO +import kotlin.math.abs +import kotlin.math.max + +/** + * Applies spatial proximity and intersection heuristics to associate + * loose text blocks (OCR) with their corresponding visual widget (YOLO). + */ +object TextAssociator { + private const val OVERLAP_THRESHOLD = 0.6 + + fun assignTextToParents(parents: List, texts: List, allBoxes: List): List { + val consumedTexts = mutableSetOf() + val updatedParents = mutableMapOf() + + for (parent in parents) { + texts.firstOrNull { text -> + !consumedTexts.contains(text) && + Rect(parent.rect).let { intersection -> + intersection.intersect(text.rect) && + (intersection.width() * intersection.height()).let { intersectionArea -> + val textArea = text.w * text.h + textArea > 0 && (intersectionArea.toFloat() / textArea.toFloat()) > OVERLAP_THRESHOLD + } + } + }?.let { + updatedParents[parent] = parent.copy(text = it.text) + consumedTexts.add(it) + } + } + + return allBoxes.mapNotNull { box -> + when { + consumedTexts.contains(box) -> null + updatedParents.containsKey(box) -> updatedParents[box] + else -> box + } + } + } + + fun assignNearbyTextToWidgets(boxes: List, availableTexts: List): List { + val consumedTexts = mutableSetOf() + val updatedWidgets = mutableMapOf() + + val labelableWidgets = boxes.filter { isLabelableWidget(it) }.sortedWith(compareBy({ it.y }, { it.x })) + + for (widget in labelableWidgets) { + val nearbyText = availableTexts + .asSequence() + .filter { it !in consumedTexts } + .filter { text -> widget.isVerticallyAlignedWith(text, tolerance = max(widget.h * 2.5, 40.0)) } + .minByOrNull { text -> widget.calculateProximityScoreTo(text) } + + if (nearbyText != null) { + val finalText = cleanWidgetText(widget, nearbyText.text) + updatedWidgets[widget] = widget.copy(text = finalText) + consumedTexts.add(nearbyText) + } + } + + return boxes.mapNotNull { box -> + when (box) { + in consumedTexts -> null + in updatedWidgets -> updatedWidgets[box] + else -> box + } + } + } + + private fun isLabelableWidget(box: ScaledBox): Boolean { + return box.label in setOf( + "radio_button_unchecked", "radio_button_checked", + "checkbox_unchecked", "checkbox_checked", + "switch_on", "switch_off" + ) + } + + private fun cleanWidgetText(widget: ScaledBox, rawText: String): String { + return if (widget.label.contains("radio", ignoreCase = true)) { + cleanTextStrippingLeadingO(rawText) + } else { + cleanTextPreservingLeadingO(rawText) + } + } + + private fun ScaledBox.isVerticallyAlignedWith(other: ScaledBox, tolerance: Double): Boolean { + return abs(this.centerY - other.centerY) < tolerance + } + + private fun ScaledBox.calculateProximityScoreTo(other: ScaledBox): Double { + val dx = this.rect.horizontalDistanceTo(other.rect).toDouble() + val dy = abs(this.centerY - other.centerY).toDouble() + return (dx * dx) + (dy * dy * 5) + } + + private fun Rect.horizontalDistanceTo(other: Rect): Int = when { + this.right < other.left -> other.left - this.right + this.left > other.right -> this.left - other.right + else -> 0 + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/WidgetAnnotationMatcher.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/WidgetAnnotationMatcher.kt new file mode 100644 index 0000000..105e49a --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/WidgetAnnotationMatcher.kt @@ -0,0 +1,131 @@ +package org.appdevforall.codeonthego.computervision.domain + +import org.appdevforall.codeonthego.computervision.domain.model.ScaledBox + +class WidgetAnnotationMatcher { + internal fun matchAnnotationsToElements( + canvasTags: List, + uiElements: List, + annotations: Map + ): Map { + val finalAnnotations = mutableMapOf() + val claimedWidgets = mutableSetOf() + + val deduplicatedTags = canvasTags + .distinctBy { WidgetTagParser.normalizeTagText(it.text) } + + val tagsByWidgetType = annotations + .mapNotNull { (tagText, annotationText) -> + val normalizedTag = WidgetTagParser.normalizeTagText(tagText) + val widgetType = getTagType(normalizedTag) ?: return@mapNotNull null + + val matchingTagBox = deduplicatedTags.find { WidgetTagParser.normalizeTagText(it.text) == normalizedTag } + + TaggedAnnotation( + normalizedTag = normalizedTag, + widgetType = widgetType, + annotation = annotationText, + tagBox = matchingTagBox + ) + } + .groupBy { it.widgetType } + + val widgetsByType = uiElements.groupBy { normalizeWidgetType(it.label) } + + for ((widgetType, taggedAnnotations) in tagsByWidgetType) { + val candidateWidgets = widgetsByType[widgetType] + ?.sortedWith(compareBy({ it.y }, { it.x })) + ?: continue + + val sortedTags = taggedAnnotations.sortedWith( + compareBy( + { WidgetTagParser.extractOrdinal(it.normalizedTag) ?: Int.MAX_VALUE }, + { it.tagBox?.y ?: Int.MAX_VALUE }, + { it.tagBox?.x ?: Int.MAX_VALUE } + ) + ) + + for (taggedAnnotation in sortedTags) { + val ordinal = WidgetTagParser.extractOrdinal(taggedAnnotation.normalizedTag) + val matchedWidget = findWidgetByOrdinalOrFallback( + ordinal = ordinal, + tagBox = taggedAnnotation.tagBox, + candidates = candidateWidgets, + claimedWidgets = claimedWidgets + ) ?: continue + + finalAnnotations[matchedWidget] = taggedAnnotation.annotation + claimedWidgets.add(matchedWidget) + } + } + + return finalAnnotations + } + + internal fun isTag(text: String): Boolean = WidgetTagParser.isTag(text) + + private fun getTagType(tag: String): String? { + return when { + tag.startsWith("B-") -> "button" + tag.startsWith("P-") -> "image_placeholder" + tag.startsWith("D-") -> "dropdown" + tag.startsWith("T-") -> "text_entry_box" + tag.startsWith("C-") -> "checkbox" + tag.startsWith("R-") -> "radio" + tag.startsWith("SW-") -> "switch" + tag.startsWith("S-") -> "slider" + else -> null + } + } + + private fun normalizeWidgetType(label: String): String = when { + label.startsWith("text_entry_box") -> "text_entry_box" + label.startsWith("button") -> "button" + label.startsWith("switch") -> "switch" + label.startsWith("checkbox") -> "checkbox" + label.startsWith("radio") -> "radio" + label.startsWith("dropdown") -> "dropdown" + label.startsWith("slider") -> "slider" + label.startsWith("image_placeholder") || label.startsWith("icon") -> "image_placeholder" + else -> label + } + + private fun findWidgetByOrdinalOrFallback( + ordinal: Int?, + tagBox: ScaledBox?, + candidates: List, + claimedWidgets: Set + ): ScaledBox? { + val available = candidates.filter { it !in claimedWidgets } + if (available.isEmpty()) return null + + if (ordinal != null) { + val oneBasedMatch = candidates.getOrNull(ordinal - 1) + if (oneBasedMatch != null && oneBasedMatch !in claimedWidgets) { + return oneBasedMatch + } + + val zeroBasedMatch = candidates.getOrNull(ordinal) + if (zeroBasedMatch != null && zeroBasedMatch !in claimedWidgets) { + return zeroBasedMatch + } + } + + if (tagBox != null) { + return available.minByOrNull { candidate -> + val verticalDistance = kotlin.math.abs(tagBox.centerY - candidate.centerY) + val horizontalDistance = kotlin.math.abs(tagBox.centerX - candidate.centerX) + (verticalDistance * 2) + horizontalDistance + } + } + + return available.minByOrNull { it.y } + } + + private data class TaggedAnnotation( + val normalizedTag: String, + val widgetType: String, + val annotation: String, + val tagBox: ScaledBox? + ) +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/WidgetTagParser.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/WidgetTagParser.kt new file mode 100644 index 0000000..1b8fa87 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/WidgetTagParser.kt @@ -0,0 +1,128 @@ +package org.appdevforall.codeonthego.computervision.domain + +/** + * Parses and normalizes raw OCR text into standardized Android widget tags. + * Handles common OCR misreads and formatting inconsistencies. + */ +internal object WidgetTagParser { + private val tagRegex = Regex("^(?i)(B|P|D|T|C|R|SW|S)-[A-Z0-9_]+$") + private val tagExtractRegex = Regex("^(?i)(B|P|D|T|C|R|SW|S|8|8W|S8)([\\s\\-_.,|/]*)([A-Z0-9_\\-]+)") + private val VALID_PREFIXES = setOf("B", "P", "D", "T", "C", "R", "SW", "S") + + fun isTag(text: String): Boolean { + val cleaned = text.trim().trimEnd('.', ',', ';', ':', '_', '|') + val match = tagExtractRegex.find(cleaned) ?: return false + + if (!isValidTagMatch(match)) return false + + val trailingText = cleaned.substring(match.range.last + 1).trim() + if (trailingText.isNotBlank() && trailingText.any { it.isLetterOrDigit() }) return false + + return normalizeTagText(cleaned).matches(tagRegex) + } + + private fun parseTagParts(match: MatchResult): Pair? { + val rawPrefix = match.groupValues[1] + val prefix = normalizePrefix(rawPrefix) + + if (prefix !in VALID_PREFIXES) return null + + var tokenRaw = match.groupValues[3].trim('-') + + val upperToken = tokenRaw.uppercase() + val remainder = upperToken.removePrefix(prefix) + + when { + upperToken.startsWith("$prefix-") || upperToken.startsWith("${prefix}_") -> { + tokenRaw = tokenRaw.substring(prefix.length + 1).trim('-') + } + upperToken.startsWith(prefix) && remainder.isNotEmpty() && remainder.all(::isNumericLikeOcrChar) -> { + tokenRaw = remainder + } + } + + val token = normalizeTagToken(tokenRaw) + return prefix to token + } + + fun normalizeTagText(text: String): String { + val cleaned = text.trim().trimEnd('.', ',', ';', ':', '_', '|') + val match = tagExtractRegex.find(cleaned) ?: return cleaned.uppercase() + + if (!isValidTagMatch(match)) return cleaned.uppercase() + + val parts = parseTagParts(match) ?: return cleaned.uppercase() + + return "${parts.first}-${parts.second}" + } + + fun extractTag(text: String): Pair? { + val cleaned = text.trim().trimEnd('.', ',', ';', ':', '_', '|') + val match = tagExtractRegex.find(cleaned) ?: return null + + if (!isValidTagMatch(match)) return null + + val parts = parseTagParts(match) ?: return null + + val finalTag = "${parts.first}-${parts.second}" + + if (!finalTag.matches(tagRegex)) return null + + val trailingText = cleaned.substring(match.range.last + 1).trim().takeIf { it.isNotBlank() } + return finalTag to trailingText + } + + private fun isValidTagMatch(match: MatchResult): Boolean { + val separator = match.groupValues[2] + val rawToken = match.groupValues[3] + + if (separator.isNotEmpty()) return true + return rawToken.all(::isNumericLikeOcrChar) + } + + private fun normalizePrefix(rawPrefix: String): String { + return rawPrefix.uppercase() + .replace(Regex("\\s+"), "") + .replace(Regex("^8$"), "B") + .replace(Regex("^(8W|S8)$"), "SW") + } + + /** + * Extracts the numeric or alphanumeric identifier part of the tag (the part after the hyphen). + */ + fun extractOrdinal(tag: String): Int? = tag.substringAfter('-', "").toIntOrNull() + + /** + * Cleans up the token suffix. If the token consists entirely of numbers or OCR artifacts, + * it converts those artifacts back to digits. + */ + private fun normalizeTagToken(rawToken: String): String { + if (rawToken.isBlank()) return rawToken + + val uppercaseToken = rawToken.uppercase().replace('-', '_') + return if (uppercaseToken.all(::isNumericLikeOcrChar)) { + normalizeOcrDigits(uppercaseToken) + } else { + uppercaseToken.replace(Regex("[^A-Z0-9_]"), "_") + } + } + + /** + * Replaces characters that are commonly misread by OCR with their intended numeric values. + */ + private fun normalizeOcrDigits(raw: String): String = + raw.replace('I', '1') + .replace('L', '1') + .replace('!', '1') + .replace('O', '0') + .replace('Z', '2') + .replace('S', '5') + .replace('B', '6') + + /** + * Determines whether a character is a digit or a letter frequently confused with a digit by OCR. + */ + private fun isNumericLikeOcrChar(char: Char): Boolean { + return char.isDigit() || char.uppercaseChar() in setOf('O', 'I', 'L', 'Z', 'S', 'B', '!') + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/YoloToXmlConverter.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/YoloToXmlConverter.kt new file mode 100644 index 0000000..9a72044 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/YoloToXmlConverter.kt @@ -0,0 +1,147 @@ +package org.appdevforall.codeonthego.computervision.domain + +import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult +import org.appdevforall.codeonthego.computervision.domain.model.ScaledBox +import org.appdevforall.codeonthego.computervision.domain.xml.AndroidXmlGenerator +import org.appdevforall.codeonthego.computervision.utils.MetadataDetector +import org.appdevforall.codeonthego.computervision.utils.buildPlaceholderOverrides +import kotlin.comparisons.compareBy + +class YoloToXmlConverter( + private val annotationMatcher: WidgetAnnotationMatcher, + private val xmlGenerator: AndroidXmlGenerator +) { + + fun generateXmlLayout( + detections: List, + annotations: Map, + selectedImagesByPlaceholderId: Map, + sourceImageWidth: Int, + sourceImageHeight: Int, + targetDpWidth: Int, + targetDpHeight: Int, + wrapInScroll: Boolean = true + ): Pair { + // 1. Filter and prepare base UI candidates + val uiCandidates = extractUiCandidates(detections) + + // 2. Scale detections to target DP dimensions + val scaledBoxes = scaleDetections(uiCandidates, sourceImageWidth, sourceImageHeight, targetDpWidth, targetDpHeight) + + // 3. Associate isolated text detections to their respective UI widgets + val associatedBoxes = associateTextToWidgets(scaledBoxes) + + // 4. Clean up and finalize the UI elements list + val uiElements = finalizeUiElements(associatedBoxes) + + // 5. Extract and scale reference tags (e.g., T-1, B-1) from the canvas + val canvasTags = extractCanvasTags(detections, sourceImageWidth, sourceImageHeight, targetDpWidth, targetDpHeight) + + // 6. Match margin annotations with the extracted UI elements + val finalAnnotations = annotationMatcher.matchAnnotationsToElements(canvasTags, uiElements, annotations) + + // 7. Sort boxes top-to-bottom, left-to-right for sequential XML rendering + val sortedBoxes = uiElements.sortedWith(compareBy({ it.y }, { it.x })) + + // 8. Prepare local drawable resources overrides for image placeholders + val selectedImageOverrides = uiElements.buildPlaceholderOverrides(selectedImagesByPlaceholderId) + + // 9. Generate final XML output + return xmlGenerator.buildXml( + boxes = sortedBoxes, + annotations = finalAnnotations, + selectedImageOverrides = selectedImageOverrides, + targetDpHeight = targetDpHeight, + wrapInScroll = wrapInScroll + ) + } + + private fun extractUiCandidates(detections: List): List { + return detections + .filter { (it.isYolo || it.label == "text") && it.label != "widget_tag" } + .filterNot { MetadataDetector.isMetadataDetection(it.label, it.text) } + .distinctBy { + if (it.label.startsWith("switch")) { + // Deduplicate switches by grouping them within a 50px vertical band + "${((it.boundingBox.top + it.boundingBox.bottom) / 2f).toInt() / 50}" + } else { + // Exact coordinate deduplication for other widgets + "${it.label}:${it.boundingBox.left}:${it.boundingBox.top}:${it.boundingBox.right}:${it.boundingBox.bottom}" + } + } + } + + private fun scaleDetections( + candidates: List, + sourceWidth: Int, + sourceHeight: Int, + targetWidth: Int, + targetHeight: Int + ): List { + return candidates.map { + DetectionScaler.scale(it, sourceWidth, sourceHeight, targetWidth, targetHeight) + } + } + + private fun associateTextToWidgets(scaledBoxes: List): List { + val parents = scaledBoxes.filter { it.label != "text" } + val initialTexts = scaledBoxes.filter { it.label == "text" && !annotationMatcher.isTag(it.text) } + + val textAssignedBoxes = TextAssociator.assignTextToParents(parents, initialTexts, scaledBoxes) + val remainingTexts = textAssignedBoxes.filter { it.label == "text" && !annotationMatcher.isTag(it.text) } + + return TextAssociator.assignNearbyTextToWidgets(textAssignedBoxes, remainingTexts) + } + + private fun finalizeUiElements(boxes: List): List { + return boxes.filter { + // Keep the widget if it's not pure text, or if it is text but not recognized as a tag. + (it.label != "text" || !annotationMatcher.isTag(it.text)) && + !MetadataDetector.isMetadataDetection(it.label, it.text) + } + } + + private fun extractCanvasTags( + detections: List, + sourceWidth: Int, + sourceHeight: Int, + targetWidth: Int, + targetHeight: Int + ): List { + val widgetTags = detections.filter { + it.label == "widget_tag" || (!it.isYolo && annotationMatcher.isTag(it.text)) + } + return widgetTags.map { + DetectionScaler.scale(it, sourceWidth, sourceHeight, targetWidth, targetHeight) + } + } + + companion object { + fun generateXmlLayout( + detections: List, + annotations: Map, + selectedImagesByPlaceholderId: Map = emptyMap(), + sourceImageWidth: Int, + sourceImageHeight: Int, + targetDpWidth: Int, + targetDpHeight: Int, + wrapInScroll: Boolean = true + ): Pair { + val matcher = WidgetAnnotationMatcher() + val generator = AndroidXmlGenerator() + + val converter = YoloToXmlConverter(matcher, generator) + + return converter.generateXmlLayout( + detections = detections, + annotations = annotations, + selectedImagesByPlaceholderId = selectedImagesByPlaceholderId, + sourceImageWidth = sourceImageWidth, + sourceImageHeight = sourceImageHeight, + targetDpWidth = targetDpWidth, + targetDpHeight = targetDpHeight, + wrapInScroll = wrapInScroll + ) + } + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/grammar/AttributeValidator.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/grammar/AttributeValidator.kt new file mode 100644 index 0000000..b022957 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/grammar/AttributeValidator.kt @@ -0,0 +1,158 @@ +package org.appdevforall.codeonthego.computervision.domain.grammar + + +import me.xdrop.fuzzywuzzy.FuzzySearch +import org.appdevforall.codeonthego.computervision.utils.extractOcrEntries + +interface AttributeValidator { + fun validate(rawValue: String): String? +} + +internal fun matchCategoricalValue(rawValue: String, allowedValues: List, threshold: Int = 70): String? { + val result = FuzzySearch.extractOne(rawValue, allowedValues) + return if (result.score >= threshold) result.string else null +} + +object PassThroughValidator : AttributeValidator { + override fun validate(rawValue: String): String = rawValue.trim() +} + +object BooleanValidator : AttributeValidator { + private val allowedValues = listOf("true", "false") + + override fun validate(rawValue: String): String? { + return matchCategoricalValue(rawValue.trim().lowercase(), allowedValues, threshold = 85) + } +} + +object DimensionValidator : AttributeValidator { + private val dimensionValues = listOf("match_parent", "wrap_content") + + override fun validate(rawValue: String): String? { + val trimmed = rawValue.trim() + if (trimmed.endsWith("dp") || trimmed.endsWith("sp") || trimmed.endsWith("px")) { + return trimmed + } + return matchCategoricalValue(trimmed, dimensionValues) + } +} + +class SpDimensionRangeValidator( + private val minSp: Int, + private val maxSp: Int +) : AttributeValidator { + private val spRegex = Regex("^(\\d+(?:\\.\\d+)?)sp$") + + override fun validate(rawValue: String): String? { + val trimmed = rawValue.trim() + + val match = spRegex.matchEntire(trimmed) ?: return null + val value = match.groupValues[1].toFloatOrNull() ?: return null + + return trimmed.takeIf { value >= minSp && value <= maxSp } + } +} + +class CategoricalValidator(private val allowedValues: List) : AttributeValidator { + override fun validate(rawValue: String): String? { + return matchCategoricalValue(rawValue.trim(), allowedValues) + } +} + +object SliderStyleValidator : AttributeValidator { + private val sliderStyles = listOf("continuous", "discrete", "material", "thick") + private val styleResourceMapping = mapOf( + "continuous" to "@style/Widget.MaterialComponents.Slider", + "discrete" to "@style/Widget.MaterialComponents.Slider.Discrete", + "material" to "@style/Widget.MaterialComponents.Slider", + "thick" to "@style/Widget.App.Slider.Thick" + ) + + override fun validate(rawValue: String): String? { + val matchedCategory = matchCategoricalValue(rawValue.trim(), sliderStyles) + return matchedCategory?.let { + styleResourceMapping[it] ?: "@style/Slider.${it.replaceFirstChar { c -> c.uppercase() }}" + } + } +} + +object EntriesValidator : AttributeValidator { + override fun validate(rawValue: String): String? { + val trimmed = rawValue.trim() + if (trimmed.startsWith("@")) return trimmed + + val content = trimmed.removeSurrounding("[", "]") + val rawItems = content.extractOcrEntries() + + val isNumericArray = isEntireArrayLikelyNumeric(rawItems) + + val cleanedItems = rawItems.map { item -> + val cleanItem = item.trim() + if (isNumericArray) { + cleanNumberArtifacts(cleanItem) + } else { + cleanTextArtifacts(cleanItem) + } + } + + return cleanedItems.joinToString(",") + } + + private fun isEntireArrayLikelyNumeric(items: List): Boolean { + if (items.isEmpty()) return false + var hasAtLeastOneDigit = false + + for (item in items) { + val cleanItem = item.trim() + if (cleanItem.isEmpty()) continue + + if (!cleanItem.matches(Regex("^[0-9oOlIzZsSbB\\s]+$"))) { + return false + } + if (cleanItem.any { it.isDigit() }) { + hasAtLeastOneDigit = true + } + } + + return hasAtLeastOneDigit + } + + private fun cleanNumberArtifacts(text: String): String { + return text + .replace(Regex("[oO]"), "0") + .replace(Regex("[lI]"), "1") + .replace(Regex("[zZ]"), "2") + .replace(Regex("[sS]"), "5") + .replace(Regex("[bB]"), "6") + .replace(Regex("\\s+"), "") + } + + private fun cleanTextArtifacts(text: String): String { + return text.replace(Regex("\\s+"), " ") + } +} + +class FlagsCategoricalValidator( + private val allowedValues: List, + private val separator: String = "|", + private val threshold: Int = 70 +) : AttributeValidator { + override fun validate(rawValue: String): String? { + val flags = rawValue.split(separator) + + val validFlags = flags.mapNotNull { flag -> + val trimmedFlag = flag.trim() + if (trimmedFlag.isEmpty()) { + null + } else { + matchCategoricalValue(trimmedFlag, allowedValues, threshold) + } + } + + return if (validFlags.isNotEmpty()) { + validFlags.distinct().joinToString(separator) + } else { + null + } + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/grammar/UiGrammarValidator.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/grammar/UiGrammarValidator.kt new file mode 100644 index 0000000..b464b5f --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/grammar/UiGrammarValidator.kt @@ -0,0 +1,32 @@ +package org.appdevforall.codeonthego.computervision.domain.grammar + +class UiGrammarValidator { + private val registry: Map = listOf( + SpinnerGrammar, + ImageViewGrammar, + EditTextGrammar, + RadioButtonGrammar, + CheckBoxGrammar, + SwitchGrammar, + RadioGroupGrammar, + SliderGrammar, + ButtonGrammar, + TextViewGrammar, + ).associateBy { it.tag } + + fun enforceGrammar(rawParsedAttributes: Map, tag: String): Map { + val grammar = registry[tag] ?: return rawParsedAttributes + val filteredMap = mutableMapOf() + + for ((key, rawValue) in rawParsedAttributes) { + val validator = grammar.attributes[key] + if (validator != null) { + val validValue = validator.validate(rawValue) + if (validValue != null) { + filteredMap[key] = validValue + } + } + } + return filteredMap + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/grammar/WidgetGrammar.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/grammar/WidgetGrammar.kt new file mode 100644 index 0000000..481e488 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/grammar/WidgetGrammar.kt @@ -0,0 +1,123 @@ +package org.appdevforall.codeonthego.computervision.domain.grammar + +import org.appdevforall.codeonthego.computervision.domain.parser.AttributeKey +import org.appdevforall.codeonthego.computervision.domain.parser.GravityValueSet +import org.appdevforall.codeonthego.computervision.domain.parser.InputTypeValueSet +import org.appdevforall.codeonthego.computervision.domain.parser.VisibilityValueSet + +interface WidgetGrammar { + val tag: String + val attributes: Map + get() = mapOf( + AttributeKey.WIDTH.xmlName to DimensionValidator, + AttributeKey.HEIGHT.xmlName to DimensionValidator, + AttributeKey.ID.xmlName to PassThroughValidator + ) +} + +interface LayoutGrammar : WidgetGrammar { + override val attributes: Map + get() = super.attributes + mapOf( + AttributeKey.LAYOUT_MARGIN.xmlName to DimensionValidator, + AttributeKey.LAYOUT_MARGIN_TOP.xmlName to DimensionValidator, + AttributeKey.LAYOUT_MARGIN_BOTTOM.xmlName to DimensionValidator, + AttributeKey.LAYOUT_MARGIN_START.xmlName to DimensionValidator, + AttributeKey.LAYOUT_MARGIN_END.xmlName to DimensionValidator, + AttributeKey.LAYOUT_GRAVITY.xmlName to CategoricalValidator(GravityValueSet.values), + AttributeKey.GRAVITY.xmlName to CategoricalValidator(GravityValueSet.values), + AttributeKey.LAYOUT_WEIGHT.xmlName to PassThroughValidator, + AttributeKey.PADDING.xmlName to DimensionValidator, + AttributeKey.VISIBILITY.xmlName to CategoricalValidator(VisibilityValueSet.values), + AttributeKey.BACKGROUND.xmlName to PassThroughValidator, + AttributeKey.BACKGROUND_TINT.xmlName to PassThroughValidator + ) +} + +interface TextGrammar : LayoutGrammar { + override val attributes: Map + get() = super.attributes + mapOf( + AttributeKey.TEXT_COLOR.xmlName to PassThroughValidator, + AttributeKey.TEXT_SIZE.xmlName to PassThroughValidator, + AttributeKey.TEXT_STYLE.xmlName to PassThroughValidator, + AttributeKey.TEXT_ALIGNMENT.xmlName to PassThroughValidator, + AttributeKey.FONT_FAMILY.xmlName to PassThroughValidator + ) +} + +interface CompoundButtonGrammar : TextGrammar { + override val attributes: Map + get() = super.attributes + mapOf( + AttributeKey.TEXT.xmlName to PassThroughValidator, + AttributeKey.CHECKED.xmlName to BooleanValidator, + AttributeKey.TEXT_SIZE.xmlName to SpDimensionRangeValidator(minSp = 8, maxSp = 32) + ) +} + + +object SpinnerGrammar : LayoutGrammar { + override val tag = "Spinner" + override val attributes = super.attributes + mapOf( + AttributeKey.TEXT.xmlName to PassThroughValidator, + AttributeKey.ENTRIES.xmlName to EntriesValidator + ) +} + +object ImageViewGrammar : LayoutGrammar { + override val tag = "ImageView" + + override val attributes = super.attributes + mapOf( + AttributeKey.SRC.xmlName to PassThroughValidator + ) +} + +object EditTextGrammar : TextGrammar { + override val tag = "EditText" + + override val attributes = super.attributes + mapOf( + AttributeKey.TEXT.xmlName to PassThroughValidator, + AttributeKey.INPUT_TYPE.xmlName to FlagsCategoricalValidator(InputTypeValueSet.values), + AttributeKey.HINT.xmlName to PassThroughValidator + ) +} + +object RadioButtonGrammar : CompoundButtonGrammar { + override val tag = "RadioButton" +} + +object CheckBoxGrammar : CompoundButtonGrammar { + override val tag = "CheckBox" +} + +object SwitchGrammar : CompoundButtonGrammar { + override val tag = "Switch" +} + +object RadioGroupGrammar : TextGrammar { + override val tag = "RadioGroup" + override val attributes = super.attributes + mapOf( + AttributeKey.ORIENTATION.xmlName to CategoricalValidator(listOf("horizontal", "vertical")), + AttributeKey.TEXT_SIZE.xmlName to SpDimensionRangeValidator(minSp = 8, maxSp = 32) + ) +} + +object SliderGrammar : LayoutGrammar { + override val tag = "com.google.android.material.slider.Slider" + override val attributes = super.attributes + mapOf( + AttributeKey.TEXT.xmlName to PassThroughValidator, + AttributeKey.STYLE.xmlName to SliderStyleValidator + ) +} + +object TextViewGrammar : TextGrammar { + override val tag = "TextView" + override val attributes = super.attributes + mapOf( + AttributeKey.TEXT.xmlName to PassThroughValidator + ) +} + +object ButtonGrammar : TextGrammar { + override val tag = "Button" + override val attributes = super.attributes + mapOf( + AttributeKey.TEXT.xmlName to PassThroughValidator + ) +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/model/DetectionResult.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/model/DetectionResult.kt new file mode 100644 index 0000000..d837aa1 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/model/DetectionResult.kt @@ -0,0 +1,11 @@ +package org.appdevforall.codeonthego.computervision.domain.model + +import android.graphics.RectF + +data class DetectionResult( + val boundingBox: RectF, + val label: String, + val score: Float, + var text: String = "", + val isYolo: Boolean = true +) \ No newline at end of file diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/model/LayoutItem.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/model/LayoutItem.kt new file mode 100644 index 0000000..5234ef0 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/model/LayoutItem.kt @@ -0,0 +1,8 @@ +package org.appdevforall.codeonthego.computervision.domain.model + +sealed interface LayoutItem { + data class SimpleView(val box: ScaledBox) : LayoutItem + data class HorizontalRow(val row: List) : LayoutItem + data class RadioGroup(val boxes: List, val orientation: String) : LayoutItem + data class CheckboxGroup(val boxes: List, val orientation: String) : LayoutItem +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/model/ScaledBox.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/model/ScaledBox.kt new file mode 100644 index 0000000..d90a18a --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/model/ScaledBox.kt @@ -0,0 +1,15 @@ +package org.appdevforall.codeonthego.computervision.domain.model + +import android.graphics.Rect + +data class ScaledBox( + val label: String, + val text: String, + val x: Int, + val y: Int, + val w: Int, + val h: Int, + val centerX: Int, + val centerY: Int, + val rect: Rect +) diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/AttributeModels.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/AttributeModels.kt new file mode 100644 index 0000000..de1ffff --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/AttributeModels.kt @@ -0,0 +1,178 @@ +package org.appdevforall.codeonthego.computervision.domain.parser + +interface AttributeValueSet { + val values: List +} + +object GravityValueSet : AttributeValueSet { + override val values = listOf( + "top", + "bottom", + "left", + "right", + "center", + "center_vertical", + "center_horizontal", + "start", + "end" + ) +} + +object DimensionValueSet : AttributeValueSet { + const val WRAP_CONTENT = "wrap_content" + const val MATCH_PARENT = "match_parent" + + override val values = listOf(WRAP_CONTENT, MATCH_PARENT) + + val matchKeywords = setOf("match", "parent") + val wrapKeywords = setOf("wrap", "content", "wrapcan") + + val allKeywords = matchKeywords + wrapKeywords +} + +object VisibilityValueSet : AttributeValueSet { + override val values = listOf( + "visible", + "invisible", + "gone" + ) +} + +object InputTypeValueSet : AttributeValueSet { + override val values = listOf( + "text", + "textPassword", + "number", + "numberDecimal", + "textEmailAddress", + "textUri", + "phone", + "textVisiblePassword", + "textPersonName", + "textCapSentences", + "textCapWords", + "textMultiLine", + "textNoSuggestions", + "date", + "time", + "datetime" + ) +} + +enum class ValueType { + RAW, + TEXT_CONTENT, + DIMENSION, + SP_DIMENSION, + COLOR, + ID, + DRAWABLE, + INTEGER, + FLOAT, + TEXT_STYLE +} + +enum class AttributeKey( + val xmlName: String, + val aliases: List, + val valueType: ValueType = ValueType.RAW +) { + WIDTH("android:layout_width", listOf("layout_width", "width"), ValueType.DIMENSION), + HEIGHT("android:layout_height", listOf("layout_height", "height"), ValueType.DIMENSION), + ID("android:id", listOf("id"), ValueType.ID), + TEXT("android:text", listOf("text"), ValueType.TEXT_CONTENT), + HINT("android:hint", listOf("hint"), ValueType.TEXT_CONTENT), + BACKGROUND("android:background", listOf("background", "bg"), ValueType.COLOR), + BACKGROUND_TINT("app:backgroundTint", listOf("backgroundtint", "background_tint", "bg_tint"), ValueType.COLOR), + SRC("android:src", listOf("src", "scr", "sre", "5rc"), ValueType.DRAWABLE), + CONTENT_DESCRIPTION("android:contentDescription", listOf("contentdescription", "content_description")), + + TEXT_SIZE("android:textSize", listOf("textsize", "text_size"), ValueType.SP_DIMENSION), + TEXT_COLOR("android:textColor", listOf("textcolor", "text_color", "color", "text_colar", "textcolar"), ValueType.COLOR), + TEXT_STYLE("android:textStyle", listOf("textstyle", "text_style"), ValueType.TEXT_STYLE), + TEXT_ALIGNMENT("android:textAlignment", listOf("textalignment", "text_alignment")), + TEXT_ALL_CAPS("android:textAllCaps", listOf("textallcaps", "text_all_caps")), + FONT_FAMILY("android:fontFamily", listOf("fontfamily", "font_family", "font")), + MAX_LINES("android:maxLines", listOf("maxlines", "max_lines"), ValueType.INTEGER), + MIN_LINES("android:minLines", listOf("minlines", "min_lines"), ValueType.INTEGER), + LINES("android:lines", listOf("lines"), ValueType.INTEGER), + SINGLE_LINE("android:singleLine", listOf("singleline", "single_line")), + ELLIPSIZE("android:ellipsize", listOf("ellipsize")), + LINE_SPACING_EXTRA("android:lineSpacingExtra", listOf("linespacingextra", "line_spacing_extra"), ValueType.SP_DIMENSION), + LETTER_SPACING("android:letterSpacing", listOf("letterspacing", "letter_spacing")), + HINT_TEXT_COLOR("android:textColorHint", listOf("hinttextcolor", "hint_text_color", "textcolorhint", "text_color_hint"), ValueType.COLOR), + IME_OPTIONS("android:imeOptions", listOf("imeoptions", "ime_options")), + + INPUT_TYPE("android:inputType", listOf("inputtype", "input_type")), + MAX_LENGTH("android:maxLength", listOf("maxlength", "max_length"), ValueType.INTEGER), + + VISIBILITY("android:visibility", listOf("visibility")), + ENABLED("android:enabled", listOf("enabled")), + CLICKABLE("android:clickable", listOf("clickable")), + FOCUSABLE("android:focusable", listOf("focusable")), + ALPHA("android:alpha", listOf("alpha")), + ELEVATION("android:elevation", listOf("elevation"), ValueType.DIMENSION), + ROTATION("android:rotation", listOf("rotation")), + + PADDING("android:padding", listOf("padding"), ValueType.DIMENSION), + PADDING_TOP("android:paddingTop", listOf("paddingtop", "padding_top"), ValueType.DIMENSION), + PADDING_BOTTOM("android:paddingBottom", listOf("paddingbottom", "padding_bottom"), ValueType.DIMENSION), + PADDING_START("android:paddingStart", listOf("paddingstart", "padding_start"), ValueType.DIMENSION), + PADDING_END("android:paddingEnd", listOf("paddingend", "padding_end"), ValueType.DIMENSION), + PADDING_LEFT("android:paddingLeft", listOf("paddingleft", "padding_left"), ValueType.DIMENSION), + PADDING_RIGHT("android:paddingRight", listOf("paddingright", "padding_right"), ValueType.DIMENSION), + + LAYOUT_MARGIN("android:layout_margin", listOf("layout_margin", "margin"), ValueType.DIMENSION), + LAYOUT_MARGIN_TOP("android:layout_marginTop", listOf("layout_margintop", "layout_margin_top", "margin_top", "margintop"), ValueType.DIMENSION), + LAYOUT_MARGIN_BOTTOM("android:layout_marginBottom", listOf("layout_marginbottom", "layout_margin_bottom", "margin_bottom", "marginbottom"), ValueType.DIMENSION), + LAYOUT_MARGIN_START("android:layout_marginStart", listOf("layout_marginstart", "layout_margin_start", "margin_start", "marginstart"), ValueType.DIMENSION), + LAYOUT_MARGIN_END("android:layout_marginEnd", listOf("layout_marginend", "layout_margin_end", "margin_end", "marginend"), ValueType.DIMENSION), + LAYOUT_MARGIN_LEFT("android:layout_marginLeft", listOf("layout_marginleft", "layout_margin_left", "margin_left"), ValueType.DIMENSION), + LAYOUT_MARGIN_RIGHT("android:layout_marginRight", listOf("layout_marginright", "layout_margin_right", "margin_right"), ValueType.DIMENSION), + + LAYOUT_WEIGHT("android:layout_weight", listOf("layout_weight", "weight"), ValueType.FLOAT), + LAYOUT_GRAVITY("android:layout_gravity", listOf("layout_gravity", "layaut_gravity")), + GRAVITY("android:gravity", listOf("gravity")), + ORIENTATION("android:orientation", listOf("orientation")), + + MIN_WIDTH("android:minWidth", listOf("minwidth", "min_width"), ValueType.DIMENSION), + MIN_HEIGHT("android:minHeight", listOf("minheight", "min_height"), ValueType.DIMENSION), + MAX_WIDTH("android:maxWidth", listOf("maxwidth", "max_width"), ValueType.DIMENSION), + MAX_HEIGHT("android:maxHeight", listOf("maxheight", "max_height"), ValueType.DIMENSION), + + SCALE_TYPE("android:scaleType", listOf("scaletype", "scale_type")), + ADJUST_VIEW_BOUNDS("android:adjustViewBounds", listOf("adjustviewbounds", "adjust_view_bounds")), + TINT("android:tint", listOf("tint"), ValueType.COLOR), + + STYLE("style", listOf("style")), + ENTRIES("android:entries", listOf("entries")), + CHECKED("android:checked", listOf("checked")), + + CARD_CORNER_RADIUS("app:cardCornerRadius", listOf("cardcornerradius", "card_corner_radius", "cornerradius", "corner_radius"), ValueType.DIMENSION), + CARD_ELEVATION("app:cardElevation", listOf("cardelevation", "card_elevation"), ValueType.DIMENSION), + CARD_BACKGROUND_COLOR("app:cardBackgroundColor", listOf("cardbackgroundcolor", "card_background_color"), ValueType.COLOR), + STROKE_COLOR("app:strokeColor", listOf("strokecolor", "stroke_color"), ValueType.COLOR), + STROKE_WIDTH("app:strokeWidth", listOf("strokewidth", "stroke_width"), ValueType.DIMENSION), + + PROGRESS("android:progress", listOf("progress"), ValueType.INTEGER), + MAX("android:max", listOf("max"), ValueType.INTEGER), + MIN("android:min", listOf("min"), ValueType.INTEGER), + VALUE_FROM("app:valueFrom", listOf("valuefrom", "value_from")), + VALUE_TO("app:valueTo", listOf("valueto", "value_to")), + STEP_SIZE("app:stepSize", listOf("stepsize", "step_size")), + TRACK_COLOR("app:trackColor", listOf("trackcolor", "track_color"), ValueType.COLOR), + THUMB_COLOR("app:thumbTint", listOf("thumbcolor", "thumb_color", "thumbtint", "thumb_tint"), ValueType.COLOR), + + FOREGROUND("android:foreground", listOf("foreground"), ValueType.COLOR), + SPINNER_MODE("android:spinnerMode", listOf("spinnermode", "spinner_mode")), + DRAWABLE_START("android:drawableStart", listOf("drawablestart", "drawable_start"), ValueType.DRAWABLE), + DRAWABLE_END("android:drawableEnd", listOf("drawableend", "drawable_end"), ValueType.DRAWABLE), + DRAWABLE_PADDING("android:drawablePadding", listOf("drawablepadding", "drawable_padding"), ValueType.DIMENSION); + + companion object { + val allAliases: List by lazy { entries.flatMap { it.aliases } } + + fun findByAlias(alias: String): AttributeKey? = + entries.firstOrNull { key -> key.aliases.any { it == alias } } + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/FuzzyAttributeParser.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/FuzzyAttributeParser.kt new file mode 100644 index 0000000..7930b63 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/FuzzyAttributeParser.kt @@ -0,0 +1,141 @@ +package org.appdevforall.codeonthego.computervision.domain.parser + +import me.xdrop.fuzzywuzzy.FuzzySearch +import org.appdevforall.codeonthego.computervision.domain.grammar.UiGrammarValidator +import org.appdevforall.codeonthego.computervision.domain.parser.sanitizer.OcrSanitizerFactory +import java.lang.StringBuilder + +object FuzzyAttributeParser { + private val grammarValidator = UiGrammarValidator() + private const val PIPE_DELIMITER = "|" + private val multipleUnderscoresRegex = Regex("_+") + private val inputTypeValues = InputTypeValueSet.values.map { it.lowercase() }.toSet() + private val sanitizer = OcrSanitizerFactory.createDefaultSanitizer() + + private val cleaners = mapOf( + ValueType.TEXT_CONTENT to TextContentCleaner, + ValueType.DIMENSION to DimensionCleaner, + ValueType.SP_DIMENSION to SpDimensionCleaner, + ValueType.COLOR to ColorCleaner, + ValueType.ID to IdCleaner, + ValueType.DRAWABLE to DrawableCleaner, + ValueType.INTEGER to NumberCleaner, + ValueType.FLOAT to FloatCleaner, + ValueType.TEXT_STYLE to TextStyleCleaner, + ValueType.RAW to ValueCleaner { it } + ) + + private val numericTypes = setOf( + ValueType.DIMENSION, + ValueType.SP_DIMENSION, + ValueType.INTEGER, + ValueType.FLOAT + ) + + fun parse(annotation: String?, tag: String): Map { + if (annotation.isNullOrBlank()) return emptyMap() + + val normalizedInput = annotation.replace(Regex("\\s+:"), ":") + val tokens = tokenizeAnnotation(normalizedInput) + + val rawAttributes = mapTokensToAttributes(tokens, tag) + val finalAttributes = grammarValidator.enforceGrammar(rawAttributes, tag) + + return finalAttributes + } + + private fun tokenizeAnnotation(annotation: String): List { + val sanitized = sanitizer.sanitize(annotation) + + return if (sanitized.contains(PIPE_DELIMITER)) { + sanitized.split(PIPE_DELIMITER).map { it.trim() }.filter { it.isNotEmpty() } + } else { + sanitized.split(Regex("[:;]|\\s+")).map { it.trim() }.filter { it.isNotEmpty() } + } + } + + private fun mapTokensToAttributes(tokens: List, tag: String): Map { + val result = mutableMapOf() + var currentKey: AttributeKey? = null + val currentValue = StringBuilder() + + for (token in tokens) { + val matchedKey = if (shouldTreatTokenAsValue(token, currentKey)) { + null + } else { + fuzzyMatchKey(token) + } + + if (matchedKey != null) { + flushAttribute(currentKey, currentValue.toString(), tag, result) + currentKey = matchedKey + currentValue.clear() + } else { + currentValue.append(token).append(" ") + } + } + + flushAttribute(currentKey, currentValue.toString(), tag, result) + return result + } + + private fun shouldTreatTokenAsValue(token: String, currentKey: AttributeKey?): Boolean { + val lowerToken = token.trim().lowercase() + + return when { + currentKey == AttributeKey.INPUT_TYPE && lowerToken in inputTypeValues -> true + currentKey?.valueType == ValueType.COLOR && isColorToken(lowerToken) -> true + currentKey?.valueType == ValueType.DIMENSION && DimensionValueSet.allKeywords.any { it in lowerToken } -> true + currentKey?.valueType in numericTypes -> lowerToken.any { it.isDigit() } + else -> false + } + } + + private fun isColorToken(token: String): Boolean { + return token.startsWith("#") || token.startsWith("@") || token in ColorCleaner.colorMap + } + + private fun flushAttribute(key: AttributeKey?, rawValue: String, tag: String, destination: MutableMap) { + if (key == null || rawValue.isBlank()) return + + val cleaner = cleaners[key.valueType] ?: ValueCleaner { it } + val cleanedValue = cleaner.clean(rawValue.trim()) + + if (cleanedValue.isNotEmpty()) { + val (xmlAttr, finalValue) = resolveXmlAttribute(key, cleanedValue, tag) + if (!destination.containsKey(xmlAttr)) { + destination[xmlAttr] = finalValue + } + } + } + + private fun fuzzyMatchKey(rawKey: String): AttributeKey? { + val normalizedKey = rawKey.lowercase() + .replace("-", "_") + .replace(".", "_") + .replace(multipleUnderscoresRegex, "_") + .replace(Regex("lay[ao0]ut"), "layout") + .replace(Regex("(?<=^|_)[lt]d(?=$|_)"), "id") + + val exactMatch = AttributeKey.findByAlias(normalizedKey) + if (exactMatch != null) return exactMatch + + if (normalizedKey.length < 2) return null + + val threshold = when { + normalizedKey.length <= 3 -> 65 + normalizedKey.length <= 6 -> 75 + else -> 80 + } + + val result = FuzzySearch.extractOne(normalizedKey, AttributeKey.allAliases) + + return if (result.score >= threshold) AttributeKey.findByAlias(result.string) else null + } + + private fun resolveXmlAttribute(key: AttributeKey, value: String, tag: String): Pair { + if (key == AttributeKey.BACKGROUND && tag == "Button") return "app:backgroundTint" to value + if (key == AttributeKey.ID) return key.xmlName to value.replace(" ", "_") + return key.xmlName to value + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/ValueCleaner.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/ValueCleaner.kt new file mode 100644 index 0000000..db5cc17 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/ValueCleaner.kt @@ -0,0 +1,5 @@ +package org.appdevforall.codeonthego.computervision.domain.parser + +fun interface ValueCleaner { + fun clean(rawValue: String): String +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/ValueCleanersImpl.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/ValueCleanersImpl.kt new file mode 100644 index 0000000..5aed95e --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/ValueCleanersImpl.kt @@ -0,0 +1,178 @@ +package org.appdevforall.codeonthego.computervision.domain.parser + +import me.xdrop.fuzzywuzzy.FuzzySearch + +internal object TextContentCleaner : ValueCleaner { + private val trailingWidgetTagRegex = Regex( + "\\s+(?:[A-Z]{1,2}\\s+)?(?:B|P|D|T|C|R|SW|S)\\s*-\\s*[A-Z0-9_]+\\s*$", + RegexOption.IGNORE_CASE + ) + private val trailingRepeatedPrefixRegex = Regex( + "\\s+(?:B|P|D|T|C|R|SW|S)\\s+(?=(?:B|P|D|T|C|R|SW|S)\\s*-\\s*[A-Z0-9_]+\\s*$)", + RegexOption.IGNORE_CASE + ) + private val multipleWhitespaceRegex = Regex("\\s+") + + override fun clean(rawValue: String): String { + return rawValue + .replace(trailingRepeatedPrefixRegex, " ") + .replace(trailingWidgetTagRegex, "") + .replace(multipleWhitespaceRegex, " ") + .trim() + } +} + + +internal object NumberCleaner : ValueCleaner { + private val ocrCharMap = mapOf( + 'O' to '0', 'A' to '0', '@' to '0', 'Q' to '0', + 'L' to '1', 'I' to '1', '|' to '1', '!' to '1', '/' to '1', '\\' to '1', + '(' to '1', ')' to '1', '[' to '1', ']' to '1', + 'Z' to '2', 'S' to '5', 'B' to '6' + ) + + override fun clean(rawValue: String): String { + val translated = rawValue.map { ocrCharMap[it.uppercaseChar()] ?: it }.joinToString("") + return Regex("-?\\d+").find(translated)?.value ?: rawValue + } +} + +internal object DimensionCleaner : ValueCleaner { + private val leadingNumberRegex = Regex("^-?\\d+") + + override fun clean(rawValue: String): String { + val trimmedValue = rawValue.trim().lowercase() + val normalized = trimmedValue.replace(" ", "_") + + if (DimensionValueSet.matchKeywords.any { it in normalized }) return DimensionValueSet.MATCH_PARENT + if (DimensionValueSet.wrapKeywords.any { it in normalized }) return DimensionValueSet.WRAP_CONTENT + + val fuzzyResult = FuzzySearch.extractOne(normalized, DimensionValueSet.values) + if (fuzzyResult.score >= 60) return fuzzyResult.string + + val unitMatch = Regex("(dp|sp|px|in|mm|pt)$").find(trimmedValue) + val originalUnit = unitMatch?.value ?: "dp" + + val firstToken = trimmedValue.substringBefore(" ") + val rawNumber = firstToken.removeSuffix(originalUnit).trim() + val numericPart = NumberCleaner.clean(rawNumber) + + val numMatch = leadingNumberRegex.find(numericPart)?.value + ?: return trimmedValue + val correctedNum = removeOcrTrailingZero(numMatch) + + return "$correctedNum$originalUnit" + } + + private fun removeOcrTrailingZero(num: String): String { + val isOcrArtifact = num.endsWith("0") && (num.toLongOrNull() ?: 0L) >= 1000L + return if (isOcrArtifact) num.dropLast(1) else num + } +} + +internal object SpDimensionCleaner : ValueCleaner { + override fun clean(rawValue: String): String { + val normalized = rawValue.lowercase().replace(" ", "").replace(Regex("(sp|5p)$"), "") + val numericPart = NumberCleaner.clean(normalized.replace("_", "")) + return if (numericPart != normalized) "${numericPart}sp" else rawValue + } +} + +internal object ColorCleaner : ValueCleaner { + val colorMap = mapOf( + "red" to "#FF0000", "rel" to "#FF0000", "rad" to "#FF0000", "reo" to "#FF0000", + "green" to "#00FF00", + "blue" to "#0000FF", "ine" to "#0000FF", "hne" to "#0000FF", "hlue" to "#0000FF", "ane" to "#0000FF", "lne" to "#0000FF", + "black" to "#000000", "white" to "#FFFFFF", "gray" to "#808080", + "grey" to "#808080", "dark_gray" to "#A9A9A9", "yellow" to "#FFFF00", + "cyan" to "#00FFFF", "magenta" to "#FF00FF", "purple" to "#800080", + "orange" to "#FFA500", "brown" to "#A52A2A", "pink" to "#FFC0CB", + "light_gray" to "#D3D3D3", "dark_blue" to "#00008B", "dark_green" to "#006400", + "dark_red" to "#8B0000", "teal" to "#008080", "navy" to "#000080", + "transparent" to "@android:color/transparent" + ) + + override fun clean(rawValue: String): String { + if (rawValue.startsWith("#") || rawValue.startsWith("@")) return rawValue + + val normalizedValue = rawValue.lowercase().replace(Regex("[^a-z_]"), "").replace(" ", "_") + + val exactColor = colorMap[normalizedValue] + if (exactColor != null) return exactColor + + val result = FuzzySearch.extractOne(normalizedValue, colorMap.keys.toList()) + return if (result.score >= 70) colorMap[result.string] ?: rawValue else rawValue + } +} + +internal object IdCleaner : ValueCleaner { + private val ID_VOCABULARY = listOf("cb", "rb", "group", "checkbox", "radio", "btn", "button", "text", "view", "img", "image", "input") + private val nonAlphanumericRegex = Regex("[^a-z0-9_]") + + override fun clean(rawValue: String): String { + val firstWord = rawValue.trim().split(Regex("\\s+")).firstOrNull() ?: rawValue + + val cleaned = firstWord.lowercase() + .replace(Regex("inm|rn|wm|nm")) { m -> if (m.value == "inm") "im" else "m" } + .replace(nonAlphanumericRegex, "_") + .replace(Regex("_+"), "_") + .trim('_') + + return normalizeKnownIdVocabulary(cleaned) + } + + private fun normalizeKnownIdVocabulary(identifier: String): String { + if (identifier.isBlank()) return identifier + return identifier.split('_').filter { it.isNotBlank() } + .flatMap(::normalizeIdToken).joinToString("_") + } + + private fun normalizeIdToken(token: String): List { + if (token.isBlank()) return emptyList() + if (token.all(Char::isDigit)) return listOf(token) + + val exactMatch = FuzzySearch.extractOne(token, ID_VOCABULARY) + if (exactMatch.score >= 80 && kotlin.math.abs(token.length - exactMatch.string.length) <= 2) { + return listOf(exactMatch.string) + } + return listOf(token) + } +} + +internal object DrawableCleaner : ValueCleaner { + override fun clean(rawValue: String): String { + if (rawValue.startsWith("@drawable/")) return rawValue + + val cleaned = rawValue.lowercase() + .replace(Regex("\\.(png|jpg|jpeg|webp|xml|svg)$"), "") + .replace(Regex("inm|rn|wm|nm")) { m -> if (m.value == "inm") "im" else "m" } + .replace(Regex("[^a-z0-9_]"), "_") + .replace(Regex("_+"), "_") + .trim('_') + + val finalCleaned = cleaned + .replace("im_age", "image") + .replace(Regex("(^|_)im($|_)"), "$1image$2") + .replace(Regex("_+"), "_") + .trim('_') + return if (finalCleaned.isEmpty()) rawValue else "@drawable/$finalCleaned" + } +} + +internal object TextStyleCleaner : ValueCleaner { + private val TEXT_STYLE_VALUES = listOf("normal", "bold", "italic", "bold|italic") + + override fun clean(rawValue: String): String { + val normalizedValue = rawValue.lowercase().replace(" ", "_") + if (normalizedValue in TEXT_STYLE_VALUES) return normalizedValue + + val result = FuzzySearch.extractOne(normalizedValue, TEXT_STYLE_VALUES) + return if (result.score >= 60) result.string else rawValue + } +} + +internal object FloatCleaner : ValueCleaner { + override fun clean(rawValue: String): String { + return Regex("-?\\d+\\.?\\d*").find(rawValue)?.value ?: rawValue + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/sanitizer/CompositeOcrSanitizer.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/sanitizer/CompositeOcrSanitizer.kt new file mode 100644 index 0000000..b26d061 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/sanitizer/CompositeOcrSanitizer.kt @@ -0,0 +1,12 @@ +package org.appdevforall.codeonthego.computervision.domain.parser.sanitizer + + +class CompositeOcrSanitizer( + private val sanitizers: List +) : OcrSanitizer { + override fun sanitize(input: String): String { + return sanitizers.fold(input) { acc, sanitizer -> + sanitizer.sanitize(acc) + } + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/sanitizer/OcrSanitizer.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/sanitizer/OcrSanitizer.kt new file mode 100644 index 0000000..56504e8 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/sanitizer/OcrSanitizer.kt @@ -0,0 +1,22 @@ +package org.appdevforall.codeonthego.computervision.domain.parser.sanitizer + + +interface OcrSanitizer { + fun sanitize(input: String): String +} + +abstract class DictionaryRegexSanitizer : OcrSanitizer { + protected abstract val rawRules: Map + + private val compiledRules: List> by lazy { + rawRules.map { (pattern, replacement) -> + Regex(pattern, RegexOption.IGNORE_CASE) to replacement + } + } + + override fun sanitize(input: String): String { + return compiledRules.fold(input) { acc, (regex, replacement) -> + acc.replace(regex, replacement) + } + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/sanitizer/OcrSanitizerFactory.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/sanitizer/OcrSanitizerFactory.kt new file mode 100644 index 0000000..ebd48f3 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/sanitizer/OcrSanitizerFactory.kt @@ -0,0 +1,16 @@ +package org.appdevforall.codeonthego.computervision.domain.parser.sanitizer + + +object OcrSanitizerFactory { + fun createDefaultSanitizer(): OcrSanitizer { + return CompositeOcrSanitizer( + listOf( + ColorSanitizer(), + TextAttributeSanitizer(), + DimensionSanitizer(), + MarginPaddingSanitizer(), + StructureSanitizer() + ) + ) + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/sanitizer/OcrSanitizerRules.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/sanitizer/OcrSanitizerRules.kt new file mode 100644 index 0000000..ea5d1b1 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/parser/sanitizer/OcrSanitizerRules.kt @@ -0,0 +1,39 @@ +package org.appdevforall.codeonthego.computervision.domain.parser.sanitizer + + +class ColorSanitizer : DictionaryRegexSanitizer() { + override val rawRules = mapOf( + "backgroundired" to "background: red", + "backgroundred" to "background: red", + "\\bback[a-z]*[-_.]?\\s*[:;]\\s*" to "background: " + ) +} + +class TextAttributeSanitizer : DictionaryRegexSanitizer() { + override val rawRules = mapOf( + "text\\s*st[yj]l?e?" to "text_style" + ) +} + +class DimensionSanitizer : DictionaryRegexSanitizer() { + override val rawRules = mapOf( + "[il]ay[a-z]*[-_.\\s]*w[a-z0-9]*\\.?\\s*[:;]\\s*" to "layout_width: ", + "[il]ay[a-z]*[-_.\\s]*hei[a-z0-9]*\\.?\\s*[:;]\\s*" to "layout_height: ", + "m?w?at[ce]h[-_\\s]?p[ar]+ent" to "match_parent" + ) +} + +class MarginPaddingSanitizer : DictionaryRegexSanitizer() { + override val rawRules = mapOf( + "layout_margin\\s+(top|bottom|start|end|left|right)" to "layout_margin_$1", + "padding\\s+(top|bottom|start|end|left|right)" to "padding_$1" + ) +} + +class StructureSanitizer : DictionaryRegexSanitizer() { + override val rawRules = mapOf( + "horizontal\\s+gravity\\s*:\\s*center\\s+layout" to "layout_gravity: center_horizontal", + "\\b[ilL][dl]\\b\\s*[:;]?" to "id: ", + "\\bS[ec][rt]\\b\\s*[:;]?" to "src: " + ) +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/usecase/GenerateXmlUC.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/usecase/GenerateXmlUC.kt new file mode 100644 index 0000000..5457059 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/usecase/GenerateXmlUC.kt @@ -0,0 +1,34 @@ +package org.appdevforall.codeonthego.computervision.domain.usecase + +import org.appdevforall.codeonthego.computervision.domain.YoloToXmlConverter +import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult +import kotlinx.coroutines.CancellationException + +/** + * Use case responsible for generating the final Android XML layout string + * and the corresponding strings.xml resource based on the detected UI elements. + */ +class GenerateXmlUC { + operator fun invoke( + detections: List, + annotations: Map, + selectedImagesByPlaceholderId: Map, + sourceImageWidth: Int, + sourceImageHeight: Int, + targetDpWidth: Int, + targetDpHeight: Int + ): Result> = runCatching { + YoloToXmlConverter.generateXmlLayout( + detections = detections, + annotations = annotations, + selectedImagesByPlaceholderId = selectedImagesByPlaceholderId, + sourceImageWidth = sourceImageWidth, + sourceImageHeight = sourceImageHeight, + targetDpWidth = targetDpWidth, + targetDpHeight = targetDpHeight, + wrapInScroll = true + ) + }.onFailure { + if (it is CancellationException) throw it + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/usecase/ImportPlaceholderImageUC.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/usecase/ImportPlaceholderImageUC.kt new file mode 100644 index 0000000..1208ca4 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/usecase/ImportPlaceholderImageUC.kt @@ -0,0 +1,23 @@ +package org.appdevforall.codeonthego.computervision.domain.usecase + +import android.net.Uri +import org.appdevforall.codeonthego.computervision.data.repository.DrawableImportHelper +import org.appdevforall.codeonthego.computervision.data.repository.ImportedDrawable + +/** + * Use case that handles copying a user-selected image from the gallery + * into the project's local drawable resources folder. + */ +class ImportPlaceholderImageUC(private val drawableImportHelper: DrawableImportHelper) { + suspend operator fun invoke( + uri: Uri, + layoutFilePath: String?, + placeholderId: String + ): Result { + return drawableImportHelper.importDrawable( + sourceUri = uri, + layoutFilePath = layoutFilePath, + fallbackName = "imported_image_$placeholderId" + ) + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/usecase/PrepareImageUC.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/usecase/PrepareImageUC.kt new file mode 100644 index 0000000..2681761 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/usecase/PrepareImageUC.kt @@ -0,0 +1,70 @@ +package org.appdevforall.codeonthego.computervision.domain.usecase + +import android.content.ContentResolver +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.graphics.Matrix +import android.net.Uri +import androidx.exifinterface.media.ExifInterface +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import org.appdevforall.codeonthego.computervision.utils.SmartBoundaryDetector +import java.io.IOException + +/** + * Use case responsible for decoding an image URI, correcting its EXIF rotation, + * and estimating the initial left and right canvas boundaries. + */ +class PrepareImageUC(private val contentResolver: ContentResolver) { + data class PreparedImage(val bitmap: Bitmap, val leftPct: Float, val rightPct: Float) + + suspend operator fun invoke(uri: Uri): Result = withContext(Dispatchers.Default) { + runCatching { + val bitmap = uriToBitmap(uri) ?: throw IllegalStateException("Failed to decode image from URI") + val rotatedBitmap = handleImageRotation(uri, bitmap) + val (leftBoundPx, rightBoundPx) = SmartBoundaryDetector.detectSmartBoundaries(rotatedBitmap) + + val widthFloat = rotatedBitmap.width.toFloat() + PreparedImage( + bitmap = rotatedBitmap, + leftPct = leftBoundPx / widthFloat, + rightPct = rightBoundPx / widthFloat + ) + }.onFailure { + if (it is CancellationException) throw it + } + } + + private fun uriToBitmap(uri: Uri): Bitmap? { + return contentResolver.openFileDescriptor(uri, "r")?.use { + BitmapFactory.decodeFileDescriptor(it.fileDescriptor) + } + } + + private fun handleImageRotation(uri: Uri, bitmap: Bitmap): Bitmap { + val orientation = try { + contentResolver.openInputStream(uri)?.use { stream -> + ExifInterface(stream).getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL) + } ?: ExifInterface.ORIENTATION_NORMAL + } catch (_: IOException) { + ExifInterface.ORIENTATION_NORMAL + } + + val matrix = Matrix().apply { + when (orientation) { + ExifInterface.ORIENTATION_ROTATE_90 -> postRotate(90f) + ExifInterface.ORIENTATION_ROTATE_180 -> postRotate(180f) + ExifInterface.ORIENTATION_ROTATE_270 -> postRotate(270f) + else -> return bitmap + } + } + return try { + val rotated = Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true) + if (rotated != bitmap) bitmap.recycle() + rotated + } catch (_: OutOfMemoryError) { + bitmap + } + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/usecase/RemovePlaceholderImageUC.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/usecase/RemovePlaceholderImageUC.kt new file mode 100644 index 0000000..8d3e9d2 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/usecase/RemovePlaceholderImageUC.kt @@ -0,0 +1,19 @@ +package org.appdevforall.codeonthego.computervision.domain.usecase + +import org.appdevforall.codeonthego.computervision.data.repository.DrawableImportHelper + +/** + * Use case that handles deleting a previously imported placeholder image + * from the project's local drawable resources folder. + */ +class RemovePlaceholderImageUC(private val drawableImportHelper: DrawableImportHelper) { + suspend operator fun invoke( + layoutFilePath: String?, + resourceName: String + ): Result { + return drawableImportHelper.deleteDrawable( + layoutFilePath = layoutFilePath, + resourceName = resourceName + ) + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/usecase/RunVisionUC.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/usecase/RunVisionUC.kt new file mode 100644 index 0000000..0bce67c --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/usecase/RunVisionUC.kt @@ -0,0 +1,67 @@ +package org.appdevforall.codeonthego.computervision.domain.usecase + +import android.graphics.Bitmap +import kotlinx.coroutines.CancellationException +import org.appdevforall.codeonthego.computervision.data.repository.VisionRepository +import org.appdevforall.codeonthego.computervision.domain.DetectionMerger +import org.appdevforall.codeonthego.computervision.domain.GenericBoxResolver +import org.appdevforall.codeonthego.computervision.domain.MarginAnnotationParser +import org.appdevforall.codeonthego.computervision.domain.RegionOcrProcessor +import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult +import org.appdevforall.codeonthego.computervision.ui.CvOperation + +/** + * Main use case that orchestrates the complete computer vision pipeline: + * 1. YOLO Detection -> 2. Region OCR -> 3. Merging -> 4. Metadata Parsing. + */ +class RunVisionUC( + private val repository: VisionRepository, + private val boxResolver: GenericBoxResolver, + private val regionOcrProcessor: RegionOcrProcessor +) { + data class VisionResult( + val detections: List, + val annotations: Map + ) + + suspend operator fun invoke( + bitmap: Bitmap, + leftPct: Float, + rightPct: Float, + onProgress: (CvOperation) -> Unit + ): Result = runCatching { + + onProgress(CvOperation.RunningYolo) + val rawDetections = repository.detectWidgets(bitmap).getOrThrow() + val resolvedDetections = boxResolver.resolve(rawDetections) + + onProgress(CvOperation.RunningOcr) + val ocrResult = regionOcrProcessor.process(bitmap, resolvedDetections, leftPct, rightPct) + + onProgress(CvOperation.MergingDetections) + val mergedDetections = DetectionMerger( + ocrResult.enrichedDetections, + ocrResult.remainingDetections, + ocrResult.fullImageTextBlocks + ).merge() + + val leftBound = bitmap.width * leftPct + val rightBound = bitmap.width * rightPct + val canvasOnlyMerged = mergedDetections.filter { detection -> + detection.isYolo || detection.boundingBox.centerX() in leftBound..rightBound + } + + val allDetections = canvasOnlyMerged + ocrResult.marginDetections + + val (canvasDetections, annotationMap) = MarginAnnotationParser.parse( + detections = allDetections, + imageWidth = bitmap.width, + leftGuidePct = leftPct, + rightGuidePct = rightPct + ) + + VisionResult(canvasDetections, annotationMap) + }.onFailure { + if (it is CancellationException) throw it + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/xml/AndroidConstants.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/xml/AndroidConstants.kt new file mode 100644 index 0000000..9cf33fc --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/xml/AndroidConstants.kt @@ -0,0 +1,30 @@ +package org.appdevforall.codeonthego.computervision.domain.xml + + +object AndroidConstants { + const val MATCH_PARENT = "match_parent" + const val WRAP_CONTENT = "wrap_content" + + const val ORIENTATION_HORIZONTAL = "horizontal" + + const val TRUE = "true" + const val FALSE = "false" + + const val DEFAULT_TEXT_SIZE = "16sp" +} + +object AndroidWidgetTags { + const val LINEAR_LAYOUT = "LinearLayout" + const val RADIO_GROUP = "RadioGroup" + + const val TEXT_VIEW = "TextView" + const val BUTTON = "Button" + const val IMAGE_VIEW = "ImageView" + const val CHECK_BOX = "CheckBox" + const val RADIO_BUTTON = "RadioButton" + const val SWITCH = "Switch" + const val EDIT_TEXT = "EditText" + const val SPINNER = "Spinner" + const val SEEK_BAR = "SeekBar" + const val VIEW = "View" +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/xml/AndroidWidget.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/xml/AndroidWidget.kt new file mode 100644 index 0000000..265fd5b --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/xml/AndroidWidget.kt @@ -0,0 +1,347 @@ +package org.appdevforall.codeonthego.computervision.domain.xml + +import org.appdevforall.codeonthego.computervision.domain.model.ScaledBox +import org.appdevforall.codeonthego.computervision.domain.parser.AttributeKey +import org.appdevforall.codeonthego.computervision.utils.extractOcrEntries + +sealed class AndroidWidget( + protected open val box: ScaledBox?, + protected val parsedAttrs: Map +) { + abstract val tag: String + var idOverride: String? = null + var extraAttrs: Map = emptyMap() + + protected open fun fallbackIdLabel() = box?.label ?: tag.lowercase() + protected open fun defaultWidth() = AndroidConstants.WRAP_CONTENT + protected open fun defaultHeight() = AndroidConstants.WRAP_CONTENT + protected open fun getChildren(): List = emptyList() + + protected abstract fun specificAttributes(): Map + + protected open fun processAttributes(context: XmlContext, id: String, attrs: Map): Map { + return attrs.mapValues { it.value.escapeXmlAttr() } + } + + fun render(context: XmlContext, indent: String) { + val resolvedId = resolveWidgetId(context) + val finalAttributes = assembleAttributes(context, resolvedId) + writeXml(context, indent, finalAttributes) + } + + protected open fun resolveWidgetId(context: XmlContext): String { + val requestedId = idOverride ?: parsedAttrs[AttributeKey.ID.xmlName]?.substringAfterLast('/') + return context.resolveId(requestedId, fallbackIdLabel()) + } + + private fun assembleAttributes(context: XmlContext, resolvedId: String): Map { + val width = parsedAttrs[AttributeKey.WIDTH.xmlName] ?: extraAttrs[AttributeKey.WIDTH.xmlName] ?: defaultWidth() + val height = parsedAttrs[AttributeKey.HEIGHT.xmlName] ?: extraAttrs[AttributeKey.HEIGHT.xmlName] ?: defaultHeight() + + val assembledAttrs = mutableMapOf( + AttributeKey.ID.xmlName to "@+id/${resolvedId.escapeXmlAttr()}", + AttributeKey.WIDTH.xmlName to width.escapeXmlAttr(), + AttributeKey.HEIGHT.xmlName to height.escapeXmlAttr() + ) + + specificAttributes().forEach { (k, v) -> assembledAttrs[k] = v.escapeXmlAttr() } + + val mergedAttrs = parsedAttrs + extraAttrs + val processedAttrs = processAttributes(context, resolvedId, mergedAttrs) + + processedAttrs.forEach { (key, value) -> + assembledAttrs.putIfAbsent(key, value) + } + + return assembledAttrs + } + + private fun writeXml(context: XmlContext, indent: String, attributes: Map) { + context.append("$indent<$tag\n") + + attributes.forEach { (key, value) -> + context.append("$indent $key=\"$value\"\n") + } + + val childWidgets = getChildren() + if (childWidgets.isEmpty()) { + context.append("$indent/>") + } else { + context.append("$indent>\n") + childWidgets.forEach { child -> + child.render(context, "$indent ") + context.appendLine() + } + context.append("$indent") + } + } + + companion object { + private val nonAlphanumericRegex = Regex("[^a-z0-9_]") + private val multipleUnderscoresRegex = Regex("_+") + + fun create(box: ScaledBox, parsedAttrs: Map): AndroidWidget { + return when (box.label) { + "text", "button", "radio_button_unchecked", "radio_button_checked" -> + TextBasedWidget(box, parsedAttrs, getTagFor(box.label)) + "checkbox_unchecked", "checkbox_checked" -> CheckBoxWidget(box, parsedAttrs) + "switch_off", "switch_on" -> SwitchWidget(box, parsedAttrs) + "text_entry_box" -> InputWidget(box, parsedAttrs) + "image_placeholder", "icon" -> ImageWidget(box, parsedAttrs) + "dropdown" -> SpinnerWidget(box, parsedAttrs) + else -> GenericWidget(box, parsedAttrs, getTagFor(box.label)) + } + } + + fun getTagFor(label: String): String = when (label) { + "text" -> AndroidWidgetTags.TEXT_VIEW + "button" -> AndroidWidgetTags.BUTTON + "image_placeholder", "icon" -> AndroidWidgetTags.IMAGE_VIEW + "checkbox_unchecked", "checkbox_checked" -> AndroidWidgetTags.CHECK_BOX + "radio_button_unchecked", "radio_button_checked" -> AndroidWidgetTags.RADIO_BUTTON + "switch_off", "switch_on" -> AndroidWidgetTags.SWITCH + "text_entry_box" -> AndroidWidgetTags.EDIT_TEXT + "dropdown" -> AndroidWidgetTags.SPINNER + "slider" -> AndroidWidgetTags.SEEK_BAR + else -> AndroidWidgetTags.VIEW + } + + internal fun sanitizeResourceName(raw: String): String { + return raw + .lowercase() + .replace('-', '_') + .replace(nonAlphanumericRegex, "_") + .replace(multipleUnderscoresRegex, "_") + .trim('_') + } + } +} + +class SpinnerWidget( + override val box: ScaledBox, parsedAttrs: Map +) : AndroidWidget(box, parsedAttrs) { + companion object { + private val placeholderEntries = setOf("year", "month", "day", "select", "choose", "dropdown") + } + + override val tag = AndroidWidgetTags.SPINNER + override fun fallbackIdLabel(): String { + val normalizedLabel = sanitizeResourceName(box.text.normalizedDropdownLabel()) + return normalizedLabel.takeIf { it.isNotBlank() }?.let { "dd_$it" } ?: "spinner" + } + override fun specificAttributes() = emptyMap() + + override fun resolveWidgetId(context: XmlContext): String { + val requestedId = idOverride ?: parsedAttrs[AttributeKey.ID.xmlName]?.substringAfterLast('/') + if (requestedId != null) return context.resolveId(requestedId, fallbackIdLabel()) + + val derivedId = fallbackIdLabel() + return derivedId + .takeUnless { it == "spinner" } + ?.let { context.resolveId(it, "spinner") } + ?: context.nextId("spinner") + } + + override fun processAttributes(context: XmlContext, id: String, attrs: Map): Map { + val processed = mutableMapOf() + val rawEntries = attrs[AttributeKey.ENTRIES.xmlName] + ?: attrs[AttributeKey.TEXT.xmlName] + ?: box.text.takeIf { it.isMeaningfulDropdownText() } + + when { + rawEntries == null -> Unit + rawEntries.trimStart().startsWith("@") -> { + processed[AttributeKey.ENTRIES.xmlName] = rawEntries.trim().escapeXmlAttr() + } + else -> rawEntries + .toSpinnerEntries() + .takeIf { items -> items.isNotEmpty() && !items.isSinglePlaceholderEntry() } + ?.let { items -> + val arrayName = "${id}_array" + context.stringArrays[arrayName] = items + processed[AttributeKey.ENTRIES.xmlName] = "@array/$arrayName" + } + } + + attrs.forEach { (key, value) -> + when { + key == AttributeKey.ENTRIES.xmlName || key == AttributeKey.TEXT.xmlName -> Unit + else -> processed[key] = value.escapeXmlAttr() + } + } + return processed + } + + private fun List.isSinglePlaceholderEntry(): Boolean { + if (size != 1) return false + return first().normalizedDropdownLabel().lowercase() in placeholderEntries + } + + private fun String.toSpinnerEntries(): List { + return this.removeTrailingDropdownGlyph().extractOcrEntries() + } + + private fun String.removeTrailingDropdownGlyph(): String { + return trim() + .replace(Regex("\\s*[▼▽▾▿⌄˅∨]$|\\s+[vV]$"), "") + .trim() + } + + private fun String.removeLeadingDropdownHint(): String { + return trim() + .replace(Regex("^[vV]\\s+"), "") + .trim() + } + + private fun String.normalizedDropdownLabel(): String { + return removeTrailingDropdownGlyph() + .removeLeadingDropdownHint() + .trim() + } + + private fun String.isMeaningfulDropdownText(): Boolean { + val cleaned = normalizedDropdownLabel() + return cleaned.isNotBlank() && !cleaned.equals("dropdown", ignoreCase = true) + } +} + +class TextBasedWidget( + override val box: ScaledBox, parsedAttrs: Map, override val tag: String +) : AndroidWidget(box, parsedAttrs) { + private val widgetTags = setOf(AndroidWidgetTags.SWITCH, AndroidWidgetTags.CHECK_BOX, AndroidWidgetTags.RADIO_BUTTON) + + override fun specificAttributes(): Map { + val attrs = mutableMapOf() + val rawViewText = parsedAttrs[AttributeKey.TEXT.xmlName] + ?: box.text.takeIf { it.isNotEmpty() && it != box.label } + ?: if (tag in widgetTags) tag else box.label + + attrs[AttributeKey.TEXT.xmlName] = rawViewText + attrs["tools:ignore"] = "HardcodedText" + + if (tag == AndroidWidgetTags.TEXT_VIEW || tag in widgetTags) { + attrs[AttributeKey.TEXT_SIZE.xmlName] = parsedAttrs[AttributeKey.TEXT_SIZE.xmlName] ?: AndroidConstants.DEFAULT_TEXT_SIZE + } + if (box.label.contains("_checked") || box.label.contains("_on")) { + attrs[AttributeKey.CHECKED.xmlName] = parsedAttrs[AttributeKey.CHECKED.xmlName] ?: AndroidConstants.TRUE + } + return attrs + } + + override fun fallbackIdLabel(): String { + return if (tag == AndroidWidgetTags.RADIO_BUTTON) "radio_button" else super.fallbackIdLabel() + } +} + +class CheckBoxWidget( + override val box: ScaledBox, parsedAttrs: Map +) : AndroidWidget(box, parsedAttrs) { + override val tag = AndroidWidgetTags.CHECK_BOX + + override fun specificAttributes(): Map { + val attrs = mutableMapOf() + val rawViewText = box.text.takeIf { it.isNotEmpty() && it != box.label } + ?: parsedAttrs[AttributeKey.TEXT.xmlName] + ?: AndroidWidgetTags.CHECK_BOX + + attrs[AttributeKey.TEXT.xmlName] = rawViewText + attrs["tools:ignore"] = "HardcodedText" + + if (box.label.contains("_checked")) { + attrs[AttributeKey.CHECKED.xmlName] = parsedAttrs[AttributeKey.CHECKED.xmlName] ?: AndroidConstants.TRUE + } + return attrs + } + + override fun fallbackIdLabel(): String = "checkbox" +} + +class SwitchWidget( + override val box: ScaledBox, parsedAttrs: Map +) : AndroidWidget(box, parsedAttrs) { + override val tag = AndroidWidgetTags.SWITCH + override fun fallbackIdLabel(): String = "switch" + + override fun specificAttributes(): Map { + val attrs = mutableMapOf() + val switchText = parsedAttrs[AttributeKey.TEXT.xmlName] ?: box.text.trim().takeIf { it.isNotEmpty() && it != box.label } ?: AndroidWidgetTags.SWITCH + + attrs[AttributeKey.TEXT.xmlName] = switchText + attrs["tools:ignore"] = "HardcodedText" + + if (box.label.contains("_on")) { + attrs[AttributeKey.CHECKED.xmlName] = parsedAttrs[AttributeKey.CHECKED.xmlName] ?: AndroidConstants.TRUE + } + return attrs + } +} + +class InputWidget( + override val box: ScaledBox, parsedAttrs: Map +) : AndroidWidget(box, parsedAttrs) { + override val tag = AndroidWidgetTags.EDIT_TEXT + + override fun specificAttributes(): Map { + val resolvedHint = parsedAttrs[AttributeKey.HINT.xmlName] ?: box.text.ifEmpty { "Enter text..." } + val resolvedInputType = parsedAttrs[AttributeKey.INPUT_TYPE.xmlName] ?: "text" + + return mapOf( + AttributeKey.HINT.xmlName to resolvedHint, + AttributeKey.INPUT_TYPE.xmlName to resolvedInputType, + "tools:ignore" to "HardcodedText" + ) + } +} + +class ImageWidget( + override val box: ScaledBox, parsedAttrs: Map +) : AndroidWidget(box, parsedAttrs) { + override val tag = AndroidWidgetTags.IMAGE_VIEW + override fun specificAttributes(): Map = mapOf( + AttributeKey.CONTENT_DESCRIPTION.xmlName to (parsedAttrs[AttributeKey.CONTENT_DESCRIPTION.xmlName] ?: box.label), + ) +} + +class GenericWidget( + override val box: ScaledBox, parsedAttrs: Map, override val tag: String +) : AndroidWidget(box, parsedAttrs) { + override fun specificAttributes() = emptyMap() +} + +abstract class AndroidViewGroup( + parsedAttrs: Map, + protected val childWidgets: List +) : AndroidWidget(null, parsedAttrs) { + override fun getChildren() = childWidgets +} + +class HorizontalRowWidget( + childWidgets: List +) : AndroidViewGroup(emptyMap(), childWidgets) { + override val tag = AndroidWidgetTags.LINEAR_LAYOUT + override fun fallbackIdLabel() = "linear_layout" + override fun defaultWidth() = AndroidConstants.MATCH_PARENT + override fun specificAttributes() = mapOf( + "android:orientation" to AndroidConstants.ORIENTATION_HORIZONTAL, + "android:baselineAligned" to AndroidConstants.FALSE + ) +} + +class RadioGroupWidget( + parsedAttrs: Map, + childWidgets: List, + private val orientation: String, + private val checkedId: String? +) : AndroidViewGroup(parsedAttrs, childWidgets) { + override val tag = AndroidWidgetTags.RADIO_GROUP + override fun fallbackIdLabel() = "radio_group" + override fun defaultWidth() = AndroidConstants.MATCH_PARENT + + override fun specificAttributes(): Map { + val attrs = mutableMapOf("android:orientation" to orientation) + if (checkedId != null) { + attrs["android:checkedButton"] = "@id/$checkedId" + } + return attrs + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/xml/AndroidXmlGenerator.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/xml/AndroidXmlGenerator.kt new file mode 100644 index 0000000..2f949bd --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/xml/AndroidXmlGenerator.kt @@ -0,0 +1,64 @@ +package org.appdevforall.codeonthego.computervision.domain.xml + +import org.appdevforall.codeonthego.computervision.domain.LayoutTreeBuilder +import org.appdevforall.codeonthego.computervision.domain.model.ScaledBox + +class AndroidXmlGenerator { + internal fun buildXml( + boxes: List, + annotations: Map, + selectedImageOverrides: Map, + targetDpHeight: Int, + wrapInScroll: Boolean + ): Pair { + val context = XmlContext() + val maxBottom = boxes.maxOfOrNull { it.y + it.h } ?: 0 + val needScroll = wrapInScroll && maxBottom > targetDpHeight + + appendHeaders(context, needScroll) + + val layoutItems = LayoutTreeBuilder.buildLayoutTree(boxes) + val renderer = LayoutRenderer(context, annotations, selectedImageOverrides = selectedImageOverrides) + + layoutItems.forEach { item -> renderer.render(item, " ") } + + appendFooters(context, needScroll) + + val layoutXml = context.toString() + val stringsXml = generateStringsResourceXml(context) + + return Pair(layoutXml, stringsXml) + } + + private fun generateStringsResourceXml(context: XmlContext): String { + if (context.stringArrays.isEmpty()) return "" + + val builder = StringBuilder() + context.stringArrays.forEach { (name, items) -> + builder.appendLine(" ") + items.forEach { item -> + builder.appendLine(" ${item.escapeXmlAttr()}") + } + builder.appendLine(" ") + } + + return builder.toString().trimEnd() + } + + private fun appendHeaders(context: XmlContext, needScroll: Boolean) { + val namespaces = """xmlns:android="http://schemas.android.com/apk/res/android" xmlns:app="http://schemas.android.com/apk/res-auto" xmlns:tools="http://schemas.android.com/tools"""" + context.appendLine("") + + if (needScroll) { + context.appendLine("") + context.appendLine(" ") + } else { + context.appendLine("") + } + context.appendLine() + } + + private fun appendFooters(context: XmlContext, needScroll: Boolean) { + context.appendLine(if (needScroll) " \n" else "") + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/xml/LayoutRenderer.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/xml/LayoutRenderer.kt new file mode 100644 index 0000000..35b558c --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/xml/LayoutRenderer.kt @@ -0,0 +1,21 @@ +package org.appdevforall.codeonthego.computervision.domain.xml + +import org.appdevforall.codeonthego.computervision.domain.model.LayoutItem +import org.appdevforall.codeonthego.computervision.domain.model.ScaledBox + +class LayoutRenderer( + private val context: XmlContext, + annotations: Map, + selectedImageOverrides: Map = emptyMap() +) { + private val widgetFactory = WidgetFactory(context, annotations, selectedImageOverrides) + + fun render(item: LayoutItem, indent: String = " ") { + val widgets = widgetFactory.createWidgets(item) + + widgets.forEach { widget -> + widget.render(context, indent) + context.appendLine() + } + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/xml/WidgetFactory.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/xml/WidgetFactory.kt new file mode 100644 index 0000000..001791b --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/xml/WidgetFactory.kt @@ -0,0 +1,194 @@ +package org.appdevforall.codeonthego.computervision.domain.xml + +import org.appdevforall.codeonthego.computervision.domain.model.LayoutItem +import org.appdevforall.codeonthego.computervision.domain.model.ScaledBox +import org.appdevforall.codeonthego.computervision.domain.parser.AttributeKey +import org.appdevforall.codeonthego.computervision.domain.parser.FuzzyAttributeParser + +class WidgetFactory( + private val context: XmlContext, + private val annotations: Map, + private val selectedImageOverrides: Map = emptyMap() +) { + private val checkboxGroupIdPattern = Regex("^cb_group_\\d+$") + private val radioChildGroupIdPatterns = listOf( + Regex("^rb_group(?:_\\d+)?(?:_|$).*"), + Regex("^radio_group(?:_\\d+)?(?:_|$).*") + ) + + fun createWidgets(item: LayoutItem): List = when (item) { + is LayoutItem.SimpleView -> createWidgetsForBox(item.box) + is LayoutItem.HorizontalRow -> createHorizontalRow(item) + is LayoutItem.RadioGroup -> createRadioGroup(item) + is LayoutItem.CheckboxGroup -> createCheckboxGroup(item) + } + + private fun createHorizontalRow(item: LayoutItem.HorizontalRow): List { + val children = item.row.flatMapIndexed { index, box -> + val extraAttrs = getMarginEndForHorizontalGap(item.row, index) + createWidgetsForBox(box, extraAttrs = extraAttrs) + } + return listOf(HorizontalRowWidget(children)) + } + + private fun createWidgetsForBox( + box: ScaledBox, + extraAttrs: Map = emptyMap(), + idOverride: String? = null, + parsedAttrsOverride: Map? = null + ): List { + val widgets = mutableListOf() + + val dropdownTitle = box.text + .takeIf { box.label == "dropdown" } + ?.let(::extractDropdownTitle) + + if (dropdownTitle != null) { + val titleBox = box.copy(label = "text", text = dropdownTitle) + + val titleAttrs = mapOf( + AttributeKey.WIDTH.xmlName to AndroidConstants.WRAP_CONTENT, + AttributeKey.HEIGHT.xmlName to AndroidConstants.WRAP_CONTENT, + AttributeKey.LAYOUT_MARGIN_BOTTOM.xmlName to "4dp", + AttributeKey.TEXT.xmlName to dropdownTitle, + AttributeKey.TEXT_STYLE.xmlName to "bold" + ) + widgets.add(createSimpleWidget(titleBox, parsedAttrsOverride = titleAttrs)) + + val baseParsedAttrs = parsedAttrsOverride?.toMutableMap() + ?: FuzzyAttributeParser.parse(annotations[box], AndroidWidget.getTagFor(box.label)).toMutableMap() + baseParsedAttrs.remove(AttributeKey.TEXT.xmlName) + + val spinnerBox = box.copy(text = "") + widgets.add(createSimpleWidget(spinnerBox, extraAttrs, idOverride, baseParsedAttrs)) + + return widgets + } + + widgets.add(createSimpleWidget(box, extraAttrs, idOverride, parsedAttrsOverride)) + return widgets + } + + private fun createRadioGroup(item: LayoutItem.RadioGroup): List { + val groupAnnotation = item.boxes.firstNotNullOfOrNull { annotations[it] } + val fullGroupAttrs = FuzzyAttributeParser.parse(groupAnnotation, "RadioGroup") + + val groupId = resolveRadioGroupId(fullGroupAttrs["android:id"]?.substringAfterLast('/')) + + val groupStructuralAttrs = setOf("android:id", "android:layout_width", "android:layout_height", "android:orientation") + val sharedAttrs = fullGroupAttrs.filterKeys { it !in groupStructuralAttrs } + + var checkedId: String? = null + + val children = item.boxes.mapIndexed { index, box -> + val parsedAttrs = (sharedAttrs + FuzzyAttributeParser.parse(annotations[box], "RadioButton")).toMutableMap() + + if (parsedAttrs["android:id"] == fullGroupAttrs["android:id"]) { + parsedAttrs.remove("android:id") + } + + val requestedId = parsedAttrs["android:id"]?.substringAfterLast('/') + val childId = if (requestedId != null && radioChildGroupIdPatterns.any { it.matches(requestedId) }) { + context.nextId("radio_button") + } else { + context.resolveId(requestedId, "radio_button") + } + + val isChecked = box.label == "radio_button_checked" || parsedAttrs["android:checked"]?.equals("true", ignoreCase = true) == true + if (isChecked) { + checkedId = childId + parsedAttrs["android:checked"] = "true" + } else { + parsedAttrs["android:checked"] = "false" + } + + val extraAttrs = if (item.orientation == "horizontal") { + getMarginEndForHorizontalGap(item.boxes, index) + } else emptyMap() + + createSimpleWidget(box, parsedAttrsOverride = parsedAttrs, idOverride = childId, extraAttrs = extraAttrs) + } + + val textStyleAttrs = setOf("android:textColor", "android:textSize", "android:textStyle", "android:fontFamily") + val groupFinalAttrs = fullGroupAttrs.filterKeys { it !in textStyleAttrs }.toMutableMap() + groupFinalAttrs["android:id"] = groupId + + return listOf(RadioGroupWidget(groupFinalAttrs, children, item.orientation, checkedId)) + } + + private fun createCheckboxGroup(item: LayoutItem.CheckboxGroup): List { + val groupAnnotation = item.boxes.firstNotNullOfOrNull { annotations[it] } + val parsedAttrs = FuzzyAttributeParser.parse(groupAnnotation, "CheckBox") + + val requestedId = parsedAttrs["android:id"]?.substringAfterLast('/') + val baseId = if (requestedId != null && checkboxGroupIdPattern.matches(requestedId)) { + context.resolveId(requestedId, "cb_group") + } else { + context.nextId("cb_group", initialIndex = 1) + } + + return item.boxes.mapIndexed { index, box -> + val suffix = ('a' + index).toString() + val childId = "${baseId}_$suffix" + + val safeAttrs = parsedAttrs.toMutableMap() + safeAttrs.remove("android:id") + + val extraAttrs = if (item.orientation == "horizontal") { + getMarginEndForHorizontalGap(item.boxes, index) + } else emptyMap() + + createSimpleWidget(box, parsedAttrsOverride = safeAttrs, idOverride = childId, extraAttrs = extraAttrs) + } + } + + private fun createSimpleWidget( + box: ScaledBox, + extraAttrs: Map = emptyMap(), + idOverride: String? = null, + parsedAttrsOverride: Map? = null + ): AndroidWidget { + val tag = AndroidWidget.getTagFor(box.label) + val rawAnnotation = annotations[box] + val parsedAttrs = parsedAttrsOverride?.toMutableMap() + ?: FuzzyAttributeParser.parse(rawAnnotation, tag).toMutableMap() + + selectedImageOverrides[box]?.let { drawableReference -> + parsedAttrs["android:src"] = drawableReference + } + + return AndroidWidget.create(box, parsedAttrs).apply { + this.idOverride = idOverride + this.extraAttrs = extraAttrs + } + } + + private fun extractDropdownTitle(rawText: String): String? { + val cleaned = rawText.trim() + .replace(Regex("\\s*[▼▽▾▿⌄˅∨]$|\\s+[vV]$"), "") + .replace(Regex("^[vV]\\s+"), "") + .trim() + + return cleaned.takeIf { it.isNotBlank() && !it.equals("dropdown", ignoreCase = true) } + } + + private fun getMarginEndForHorizontalGap(boxes: List, currentIndex: Int): Map { + if (currentIndex >= boxes.lastIndex) return emptyMap() + val currentBox = boxes[currentIndex] + val nextBox = boxes[currentIndex + 1] + val gap = maxOf(0, nextBox.x - (currentBox.x + currentBox.w)) + return mapOf("android:layout_marginEnd" to "${gap}dp") + } + + private fun resolveRadioGroupId(requestedId: String?): String { + var cleanId = requestedId + if (requestedId != null) { + val normalizedId = requestedId.lowercase() + when { + normalizedId.startsWith("radio_grou") -> cleanId = "radio_group" + normalizedId.startsWith("rb_grou") || normalizedId.startsWith("rb_group") -> cleanId = "rb_group" + } + } + return context.resolveId(cleanId, "radio_group") + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/xml/XmlContext.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/xml/XmlContext.kt new file mode 100644 index 0000000..737e60f --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/domain/xml/XmlContext.kt @@ -0,0 +1,56 @@ +package org.appdevforall.codeonthego.computervision.domain.xml + +class XmlContext( + val builder: StringBuilder = StringBuilder(), + private val counters: MutableMap = mutableMapOf() +) { + private val usedIds = mutableSetOf() + val stringArrays = mutableMapOf>() + + fun nextId(label: String, initialIndex: Int = 0): String { + val safeLabel = label.replace(Regex("[^a-zA-Z0-9_]"), "_") + + var count = counters.getOrDefault(safeLabel, initialIndex - 1) + var newId: String + + do { + count++ + newId = "${safeLabel}_$count" + } while (usedIds.contains(newId)) + + counters[safeLabel] = count + usedIds.add(newId) + + return newId + } + + fun registerId(id: String) { + usedIds.add(id) + } + + fun resolveId(requestedId: String?, fallbackLabel: String): String { + return if (requestedId != null) { + registerId(requestedId) + requestedId + } else { + nextId(fallbackLabel) + } + } + + fun appendLine(text: String = "") { + builder.appendLine(text) + } + + fun append(text: String) { + builder.append(text) + } + + override fun toString(): String = builder.toString() +} + +fun String.escapeXmlAttr(): String = this.trim() + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace("\"", """) + .replace("'", "'") diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/ui/ComputerVisionEvent.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/ui/ComputerVisionEvent.kt new file mode 100644 index 0000000..857a4d5 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/ui/ComputerVisionEvent.kt @@ -0,0 +1,18 @@ +package org.appdevforall.codeonthego.computervision.ui + +import android.net.Uri + +sealed class ComputerVisionEvent { + data class ImageSelected(val uri: Uri) : ComputerVisionEvent() + data class ImageCaptured(val uri: Uri, val success: Boolean) : ComputerVisionEvent() + object RunDetection : ComputerVisionEvent() + object UpdateLayoutFile : ComputerVisionEvent() + object ConfirmUpdate : ComputerVisionEvent() + object SaveToDownloads : ComputerVisionEvent() + object OpenImagePicker : ComputerVisionEvent() + object RequestCameraPermission : ComputerVisionEvent() + data class UpdateGuides(val leftPct: Float, val rightPct: Float) : ComputerVisionEvent() + data class ImagePlaceholderTapped(val imageX: Float, val imageY: Float) : ComputerVisionEvent() + data class PlaceholderImageSelected(val uri: Uri) : ComputerVisionEvent() + data class RemovePlaceholderImage(val placeholderId: String) : ComputerVisionEvent() +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/ui/ComputerVisionUiState.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/ui/ComputerVisionUiState.kt new file mode 100644 index 0000000..69f010b --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/ui/ComputerVisionUiState.kt @@ -0,0 +1,61 @@ +package org.appdevforall.codeonthego.computervision.ui + +import android.graphics.Bitmap +import android.net.Uri +import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult + +data class ComputerVisionUiState( + val currentBitmap: Bitmap? = null, + val imageUri: Uri? = null, + val detections: List = emptyList(), + val visualizedBitmap: Bitmap? = null, + val layoutFilePath: String? = null, + val layoutFileName: String? = null, + val isModelInitialized: Boolean = false, + val currentOperation: CvOperation = CvOperation.Idle, + val leftGuidePct: Float = 0.2f, + val rightGuidePct: Float = 0.8f, + val parsedAnnotations: Map = emptyMap(), // Replaced old marginAnnotations + val pendingImagePlaceholderId: String? = null, + val selectedImagesByPlaceholderId: Map = emptyMap() +) { + val hasImage: Boolean + get() = currentBitmap != null + + val hasDetections: Boolean + get() = detections.isNotEmpty() + + val canRunDetection: Boolean + get() = hasImage && isModelInitialized && currentOperation == CvOperation.Idle + + val canGenerateXml: Boolean + get() = hasDetections && currentOperation == CvOperation.Idle +} + +sealed class CvOperation { + object Idle : CvOperation() + object InitializingModel : CvOperation() + object RunningYolo : CvOperation() + object RunningOcr : CvOperation() + object MergingDetections : CvOperation() + object GeneratingXml : CvOperation() + object SavingFile : CvOperation() +} + +sealed class ComputerVisionEffect { + object OpenImagePicker : ComputerVisionEffect() + object RequestCameraPermission : ComputerVisionEffect() + data class LaunchCamera(val outputUri: Uri) : ComputerVisionEffect() + data class ShowToast(val messageResId: Int) : ComputerVisionEffect() + data class ShowError(val message: String) : ComputerVisionEffect() + data class ShowConfirmDialog(val fileName: String) : ComputerVisionEffect() + data class ReturnXmlResult(val layoutXml: String, val stringsXml: String) : ComputerVisionEffect() + data class FileSaved(val fileName: String) : ComputerVisionEffect() + object NavigateBack : ComputerVisionEffect() + object OpenPlaceholderImagePicker : ComputerVisionEffect() +} + +data class SelectedImportedImage( + val resourceName: String, + val drawableReference: String +) diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/ui/GuidelinesView.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/ui/GuidelinesView.kt new file mode 100644 index 0000000..d1ea9ff --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/ui/GuidelinesView.kt @@ -0,0 +1,143 @@ +package org.appdevforall.codeonthego.computervision.ui + +import android.content.Context +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Matrix +import android.graphics.Paint +import android.graphics.RectF +import android.util.AttributeSet +import android.view.MotionEvent +import android.view.View + +class GuidelinesView @JvmOverloads constructor( + context: Context, + attrs: AttributeSet? = null, + defStyleAttr: Int = 0 +) : View(context, attrs, defStyleAttr) { + + private val linePaint = Paint().apply { + color = Color.RED + strokeWidth = 5f + alpha = 180 + } + + private val textPaint = Paint().apply { + color = Color.WHITE + textSize = 40f + textAlign = Paint.Align.CENTER + setShadowLayer(5.0f, 0f, 0f, Color.BLACK) + } + + private var leftGuidelinePct = 0.2f + private var rightGuidelinePct = 0.8f + private var draggingLine: Int = -1 // -1: none, 0: left, 1: right + private val minDistancePct = 0.05f + + private val viewMatrix = Matrix() + private val imageRect = RectF() + + var onGuidelinesChanged: ((Float, Float) -> Unit)? = null + + fun updateMatrix(matrix: Matrix) { + viewMatrix.set(matrix) + invalidate() + } + + fun setImageDimensions(width: Int, height: Int) { + imageRect.set(0f, 0f, width.toFloat(), height.toFloat()) + invalidate() + } + + fun updateGuidelines(leftPct: Float, rightPct: Float) { + leftGuidelinePct = leftPct + rightGuidelinePct = rightPct + invalidate() + } + + override fun onDraw(canvas: Canvas) { + super.onDraw(canvas) + if (imageRect.isEmpty) return + + // 1. Define line X coordinates in the image's coordinate system + val leftLineImageX = imageRect.width() * leftGuidelinePct + val rightLineImageX = imageRect.width() * rightGuidelinePct + + // We only need the X coordinates for mapping + val linePoints = floatArrayOf(leftLineImageX, 0f, rightLineImageX, 0f) + + // 2. Map image X coordinates to screen X coordinates + viewMatrix.mapPoints(linePoints) + val leftLineScreenX = linePoints[0] + val rightLineScreenX = linePoints[2] + + // 3. Draw the lines across the full height of the view (screen) + canvas.drawLine(leftLineScreenX, 0f, leftLineScreenX, height.toFloat(), linePaint) + canvas.drawLine(rightLineScreenX, 0f, rightLineScreenX, height.toFloat(), linePaint) + + // 4. Draw the labels at the bottom of the screen + val labelY = height - 60f + canvas.drawText("Left Margin", leftLineScreenX, labelY, textPaint) + canvas.drawText("Right Margin", rightLineScreenX, labelY, textPaint) + } + + override fun onTouchEvent(event: MotionEvent): Boolean { + if (imageRect.isEmpty) return false + + // Map screen touch coordinates to image coordinates for dragging + val points = floatArrayOf(event.x, event.y) + val invertedMatrix = Matrix() + viewMatrix.invert(invertedMatrix) + invertedMatrix.mapPoints(points) + val mappedX = points[0] + + when (event.action) { + MotionEvent.ACTION_DOWN -> { + val leftLineImageX = imageRect.width() * leftGuidelinePct + val rightLineImageX = imageRect.width() * rightGuidelinePct + + val screenPointLeft = mapImageCoordsToScreenCoords(leftLineImageX, imageRect.centerY()) + val screenPointRight = mapImageCoordsToScreenCoords(rightLineImageX, imageRect.centerY()) + + if (isCloseTo(event.x, screenPointLeft[0])) { + draggingLine = 0 + return true + } else if (isCloseTo(event.x, screenPointRight[0])) { + draggingLine = 1 + return true + } + } + MotionEvent.ACTION_MOVE -> { + if (draggingLine != -1) { + val newPct = (mappedX / imageRect.width()).coerceIn(0f, 1f) + if (draggingLine == 0) { + if (newPct < rightGuidelinePct - minDistancePct) { + leftGuidelinePct = newPct + } + } else { // draggingLine == 1 + if (newPct > leftGuidelinePct + minDistancePct) { + rightGuidelinePct = newPct + } + } + onGuidelinesChanged?.invoke(leftGuidelinePct, rightGuidelinePct) + invalidate() + return true + } + } + MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL -> { + draggingLine = -1 + } + } + return false + } + + private fun mapImageCoordsToScreenCoords(imageX: Float, imageY: Float): FloatArray { + val point = floatArrayOf(imageX, imageY) + viewMatrix.mapPoints(point) + return point + } + + private fun isCloseTo(x: Float, lineX: Float, threshold: Float = 40f): Boolean { + return Math.abs(x - lineX) < threshold + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/ui/ZoomableImageView.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/ui/ZoomableImageView.kt new file mode 100644 index 0000000..0f1b0aa --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/ui/ZoomableImageView.kt @@ -0,0 +1,206 @@ +package org.appdevforall.codeonthego.computervision.ui + +import android.content.Context +import android.graphics.Matrix +import android.graphics.PointF +import android.graphics.RectF +import android.graphics.drawable.Drawable +import android.util.AttributeSet +import android.view.MotionEvent +import android.view.ScaleGestureDetector +import androidx.appcompat.widget.AppCompatImageView +import kotlin.math.abs + +class ZoomableImageView @JvmOverloads constructor( + context: Context, + attrs: AttributeSet? = null, + defStyleAttr: Int = 0 +) : AppCompatImageView(context, attrs, defStyleAttr) { + + private val currentMatrix = Matrix() + private val matrixValues = FloatArray(9) + private val scaleGestureDetector: ScaleGestureDetector + private var last = PointF() + private var start = PointF() + private var minScale = 1f + private var maxScale = 4f + private var mode = NONE + + var onMatrixChangeListener: ((Matrix) -> Unit)? = null + var onImageTapListener: ((Float, Float) -> Boolean)? = null + + init { + super.setClickable(true) + scaleGestureDetector = ScaleGestureDetector(context, ScaleListener()) + scaleType = ScaleType.MATRIX + imageMatrix = currentMatrix + } + + override fun onTouchEvent(event: MotionEvent): Boolean { + scaleGestureDetector.onTouchEvent(event) + val curr = PointF(event.x, event.y) + when (event.action) { + MotionEvent.ACTION_DOWN -> { + last.set(curr) + start.set(last) + mode = DRAG + } + MotionEvent.ACTION_MOVE -> if (mode == DRAG) { + val deltaX = curr.x - last.x + val deltaY = curr.y - last.y + val fixTransX = getFixDragTrans(deltaX, width.toFloat(), origWidth * scale) + val fixTransY = getFixDragTrans(deltaY, height.toFloat(), origHeight * scale) + currentMatrix.postTranslate(fixTransX, fixTransY) + fixTrans() + last.set(curr.x, curr.y) + } + MotionEvent.ACTION_UP -> { + mode = NONE + val xDiff = abs(curr.x - start.x).toInt() + val yDiff = abs(curr.y - start.y).toInt() + if (xDiff < CLICK && yDiff < CLICK) { + val mappedPoint = mapViewPointToImage(event.x, event.y) + val consumed = mappedPoint?.let { + onImageTapListener?.invoke(it.x, it.y) + } ?: false + + if (!consumed) { + performClick() + } + } + } + MotionEvent.ACTION_POINTER_UP -> mode = NONE + } + imageMatrix = currentMatrix + onMatrixChangeListener?.invoke(currentMatrix) + return true + } + + fun mapViewPointToImage(x: Float, y: Float): PointF? { + val drawable = drawable ?: return null + val points = floatArrayOf(x, y) + val inverseMatrix = Matrix() + + if (!currentMatrix.invert(inverseMatrix)) return null + inverseMatrix.mapPoints(points) + + val imageX = points[0] + val imageY = points[1] + + return PointF(imageX, imageY).takeIf { + it.x in 0f..drawable.intrinsicWidth.toFloat() && + it.y in 0f..drawable.intrinsicHeight.toFloat() + } + } + + override fun performClick(): Boolean { + super.performClick() + return true + } + + override fun setImageDrawable(drawable: Drawable?) { + super.setImageDrawable(drawable) + fitToScreen() + } + + override fun onSizeChanged(w: Int, h: Int, oldw: Int, oldh: Int) { + super.onSizeChanged(w, h, oldw, oldh) + fitToScreen() + } + + private fun fitToScreen() { + if (drawable == null || width == 0 || height == 0) { + return + } + + val drawableWidth = origWidth + val drawableHeight = origHeight + + val viewRect = RectF(0f, 0f, width.toFloat(), height.toFloat()) + val drawableRect = RectF(0f, 0f, drawableWidth, drawableHeight) + + currentMatrix.setRectToRect(drawableRect, viewRect, Matrix.ScaleToFit.CENTER) + imageMatrix = currentMatrix + onMatrixChangeListener?.invoke(currentMatrix) // Notify listener of the change + + currentMatrix.getValues(matrixValues) + minScale = matrixValues[Matrix.MSCALE_X] + maxScale = minScale * 4 + } + + private inner class ScaleListener : ScaleGestureDetector.SimpleOnScaleGestureListener() { + override fun onScaleBegin(detector: ScaleGestureDetector): Boolean { + mode = ZOOM + return true + } + + override fun onScale(detector: ScaleGestureDetector): Boolean { + var mScaleFactor = detector.scaleFactor + val origScale = scale + var currentScale = origScale * mScaleFactor + if (currentScale > maxScale) { + mScaleFactor = maxScale / origScale + } else if (currentScale < minScale) { + mScaleFactor = minScale / origScale + } + currentScale = origScale * mScaleFactor + if (origWidth * currentScale <= width || origHeight * currentScale <= height) { + currentMatrix.postScale(mScaleFactor, mScaleFactor, width / 2f, height / 2f) + } else { + currentMatrix.postScale(mScaleFactor, mScaleFactor, detector.focusX, detector.focusY) + } + fixTrans() + return true + } + } + + private fun fixTrans() { + currentMatrix.getValues(matrixValues) + val transX = matrixValues[Matrix.MTRANS_X] + val transY = matrixValues[Matrix.MTRANS_Y] + val fixTransX = getFixTrans(transX, width.toFloat(), origWidth * scale) + val fixTransY = getFixTrans(transY, height.toFloat(), origHeight * scale) + if (fixTransX != 0f || fixTransY != 0f) { + currentMatrix.postTranslate(fixTransX, fixTransY) + } + } + + private fun getFixTrans(trans: Float, viewSize: Float, contentSize: Float): Float { + val minTrans: Float + val maxTrans: Float + if (contentSize <= viewSize) { + minTrans = 0f + maxTrans = viewSize - contentSize + } else { + minTrans = viewSize - contentSize + maxTrans = 0f + } + if (trans < minTrans) return -trans + minTrans + return if (trans > maxTrans) -trans + maxTrans else 0f + } + + private fun getFixDragTrans(delta: Float, viewSize: Float, contentSize: Float): Float { + return if (contentSize <= viewSize) { + 0f + } else delta + } + + private val scale: Float + get() { + currentMatrix.getValues(matrixValues) + return matrixValues[Matrix.MSCALE_X] + } + + private val origWidth: Float + get() = drawable?.intrinsicWidth?.toFloat() ?: 0f + + private val origHeight: Float + get() = drawable?.intrinsicHeight?.toFloat() ?: 0f + + companion object { + private const val NONE = 0 + private const val DRAG = 1 + private const val ZOOM = 2 + private const val CLICK = 3 + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/ui/viewmodel/ComputerVisionViewModel.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/ui/viewmodel/ComputerVisionViewModel.kt new file mode 100644 index 0000000..b81e315 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/ui/viewmodel/ComputerVisionViewModel.kt @@ -0,0 +1,313 @@ +package org.appdevforall.codeonthego.computervision.ui.viewmodel + +import android.net.Uri +import android.util.Log +import androidx.lifecycle.ViewModel +import androidx.lifecycle.viewModelScope +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.receiveAsFlow +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.launch +import com.appdevforall.sketchtoui.plugin.R +import org.appdevforall.codeonthego.computervision.data.repository.DrawableImportHelper +import org.appdevforall.codeonthego.computervision.data.repository.VisionRepository +import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult +import org.appdevforall.codeonthego.computervision.domain.usecase.GenerateXmlUC +import org.appdevforall.codeonthego.computervision.domain.usecase.ImportPlaceholderImageUC +import org.appdevforall.codeonthego.computervision.domain.usecase.PrepareImageUC +import org.appdevforall.codeonthego.computervision.domain.usecase.RemovePlaceholderImageUC +import org.appdevforall.codeonthego.computervision.domain.usecase.RunVisionUC +import org.appdevforall.codeonthego.computervision.ui.ComputerVisionEffect +import org.appdevforall.codeonthego.computervision.ui.ComputerVisionEvent +import org.appdevforall.codeonthego.computervision.ui.ComputerVisionUiState +import org.appdevforall.codeonthego.computervision.ui.CvOperation +import org.appdevforall.codeonthego.computervision.ui.SelectedImportedImage +import org.appdevforall.codeonthego.computervision.utils.CvAnalyticsUtil +import org.appdevforall.codeonthego.computervision.utils.getSortedPlaceholders + +class ComputerVisionViewModel( + private val repository: VisionRepository, + private val prepareImageUC: PrepareImageUC, + private val runVisionUC: RunVisionUC, + private val generateXmlUC: GenerateXmlUC, + private val importPlaceholderImageUC: ImportPlaceholderImageUC, + private val removePlaceholderImageUC: RemovePlaceholderImageUC, + layoutFilePath: String?, + layoutFileName: String? +) : ViewModel() { + + private val _uiState = MutableStateFlow( + ComputerVisionUiState( + layoutFilePath = layoutFilePath, + layoutFileName = layoutFileName + ) + ) + val uiState: StateFlow = _uiState.asStateFlow() + + private val _uiEffect = Channel() + val uiEffect = _uiEffect.receiveAsFlow() + + init { + initModel() + } + + fun onEvent(event: ComputerVisionEvent) { + when (event) { + is ComputerVisionEvent.ImageSelected -> { + CvAnalyticsUtil.trackImageSelected(fromCamera = false) + loadImageFromUri(event.uri) + } + is ComputerVisionEvent.ImageCaptured -> handleCameraResult(event.uri, event.success) + ComputerVisionEvent.RunDetection -> runDetection() + ComputerVisionEvent.UpdateLayoutFile -> showUpdateConfirmation() + ComputerVisionEvent.ConfirmUpdate -> performLayoutUpdate() + ComputerVisionEvent.SaveToDownloads -> saveXmlToDownloads() + ComputerVisionEvent.OpenImagePicker -> viewModelScope.launch { _uiEffect.send(ComputerVisionEffect.OpenImagePicker) } + ComputerVisionEvent.RequestCameraPermission -> viewModelScope.launch { _uiEffect.send(ComputerVisionEffect.RequestCameraPermission) } + is ComputerVisionEvent.UpdateGuides -> updateGuides(event.leftPct, event.rightPct) + is ComputerVisionEvent.ImagePlaceholderTapped -> handleImagePlaceholderTap(event.imageX, event.imageY) + is ComputerVisionEvent.PlaceholderImageSelected -> handlePlaceholderImageSelected(event.uri) + is ComputerVisionEvent.RemovePlaceholderImage -> removePlaceholderImage(event.placeholderId) + } + } + + private fun initModel() { + viewModelScope.launch { + _uiState.update { it.copy(currentOperation = CvOperation.InitializingModel) } + repository.initModel() + .onSuccess { _uiState.update { it.copy(isModelInitialized = true, currentOperation = CvOperation.Idle) } } + .onFailure { exception -> + Log.e(TAG, "Model initialization failed", exception) + _uiState.update { it.copy(currentOperation = CvOperation.Idle) } + _uiEffect.send(ComputerVisionEffect.ShowError("Model initialization failed: ${exception.message}")) + } + } + } + + fun onScreenStarted() { + CvAnalyticsUtil.trackScreenOpened() + } + + private fun loadImageFromUri(uri: Uri) { + viewModelScope.launch { + prepareImageUC(uri).onSuccess { prepared -> + _uiState.update { + it.copy( + currentBitmap = prepared.bitmap, + imageUri = uri, + detections = emptyList(), + visualizedBitmap = null, + leftGuidePct = prepared.leftPct, + rightGuidePct = prepared.rightPct, + parsedAnnotations = emptyMap(), + pendingImagePlaceholderId = null, + selectedImagesByPlaceholderId = emptyMap() + ) + } + }.onFailure { e -> + Log.e(TAG, "Error loading image", e) + _uiEffect.send(ComputerVisionEffect.ShowError("Failed to load image: ${e.message}")) + } + } + } + + private fun handleCameraResult(uri: Uri, success: Boolean) { + if (success) { + CvAnalyticsUtil.trackImageSelected(fromCamera = true) + loadImageFromUri(uri) + } else { + viewModelScope.launch { _uiEffect.send(ComputerVisionEffect.ShowToast(R.string.msg_image_capture_cancelled)) } + } + } + + private fun runDetection() { + val state = _uiState.value + val bitmap = state.currentBitmap ?: run { + viewModelScope.launch { _uiEffect.send(ComputerVisionEffect.ShowToast(R.string.msg_select_image_first)) } + return + } + + viewModelScope.launch { + CvAnalyticsUtil.trackDetectionStarted() + val startTime = System.currentTimeMillis() + + runVisionUC(bitmap, state.leftGuidePct, state.rightGuidePct) { operation -> + _uiState.update { it.copy(currentOperation = operation) } + }.onSuccess { result -> + CvAnalyticsUtil.trackDetectionCompleted(true, result.detections.size, System.currentTimeMillis() - startTime) + _uiState.update { + it.copy( + detections = result.detections, + parsedAnnotations = result.annotations, + currentOperation = CvOperation.Idle, + pendingImagePlaceholderId = null, + selectedImagesByPlaceholderId = emptyMap() + ) + } + }.onFailure { exception -> + Log.e(TAG, "Detection failed", exception) + CvAnalyticsUtil.trackDetectionCompleted(false, 0, System.currentTimeMillis() - startTime) + _uiState.update { it.copy(currentOperation = CvOperation.Idle) } + _uiEffect.send(ComputerVisionEffect.ShowError("Detection failed: ${exception.message}")) + } + } + } + + private fun showUpdateConfirmation() { + val state = _uiState.value + if (!state.hasDetections || state.currentBitmap == null) { + viewModelScope.launch { _uiEffect.send(ComputerVisionEffect.ShowToast(R.string.msg_run_detection_first)) } + return + } + viewModelScope.launch { _uiEffect.send(ComputerVisionEffect.ShowConfirmDialog(state.layoutFileName ?: "layout.xml")) } + } + + private fun performLayoutUpdate() { + val state = _uiState.value + if (!state.hasDetections || state.currentBitmap == null) return + + viewModelScope.launch { + _uiState.update { it.copy(currentOperation = CvOperation.GeneratingXml) } + + generateXml(state) + .onSuccess { (layoutXml, stringsXml) -> + CvAnalyticsUtil.trackXmlGenerated(state.detections.size) + CvAnalyticsUtil.trackXmlExported(toDownloads = false) + _uiState.update { it.copy(currentOperation = CvOperation.Idle) } + _uiEffect.send(ComputerVisionEffect.ReturnXmlResult(layoutXml, stringsXml)) + }.onFailure { exception -> + Log.e(TAG, "XML generation failed", exception) + _uiState.update { it.copy(currentOperation = CvOperation.Idle) } + _uiEffect.send(ComputerVisionEffect.ShowError("XML generation failed: ${exception.message}")) + } + } + } + + private fun saveXmlToDownloads() { + val state = _uiState.value + if (!state.hasDetections || state.currentBitmap == null) { + viewModelScope.launch { _uiEffect.send(ComputerVisionEffect.ShowToast(R.string.msg_run_detection_first)) } + return + } + + viewModelScope.launch { + _uiState.update { it.copy(currentOperation = CvOperation.GeneratingXml) } + + generateXml(state).onSuccess { (layoutXml, _) -> + CvAnalyticsUtil.trackXmlGenerated(state.detections.size) + CvAnalyticsUtil.trackXmlExported(toDownloads = true) + _uiState.update { it.copy(currentOperation = CvOperation.SavingFile) } + _uiState.update { it.copy(currentOperation = CvOperation.Idle) } + + _uiEffect.send(ComputerVisionEffect.FileSaved(layoutXml)) + }.onFailure { exception -> + Log.e(TAG, "XML generation failed", exception) + _uiState.update { it.copy(currentOperation = CvOperation.Idle) } + _uiEffect.send(ComputerVisionEffect.ShowError("XML generation failed: ${exception.message}")) + } + } + } + + private fun updateGuides(leftPct: Float, rightPct: Float) { + val clampedLeft = leftPct.coerceIn(0f, 1f) + val clampedRight = rightPct.coerceIn(0f, 1f) + + _uiState.update { + it.copy( + leftGuidePct = minOf(clampedLeft, clampedRight), + rightGuidePct = maxOf(clampedLeft, clampedRight) + ) + } + } + + private fun generateXml(state: ComputerVisionUiState): Result> { + val bitmap = state.currentBitmap ?: return Result.failure(IllegalStateException("No bitmap available")) + return generateXmlUC( + detections = state.detections, + annotations = state.parsedAnnotations, + selectedImagesByPlaceholderId = state.selectedImagesByPlaceholderId.mapValues { it.value.drawableReference }, + sourceImageWidth = bitmap.width, + sourceImageHeight = bitmap.height, + targetDpWidth = TARGET_DP_WIDTH, + targetDpHeight = TARGET_DP_HEIGHT + ) + } + + private fun handleImagePlaceholderTap(imageX: Float, imageY: Float) { + val placeholder = findImagePlaceholderAt(imageX, imageY) ?: return + val placeholderId = resolvePlaceholderId(placeholder) + + _uiState.update { it.copy(pendingImagePlaceholderId = placeholderId) } + viewModelScope.launch { _uiEffect.send(ComputerVisionEffect.OpenPlaceholderImagePicker) } + } + + private fun handlePlaceholderImageSelected(uri: Uri) { + val state = _uiState.value + val placeholderId = state.pendingImagePlaceholderId ?: return + + viewModelScope.launch { + importPlaceholderImageUC(uri, state.layoutFilePath, placeholderId) + .onSuccess { importedDrawable -> + _uiState.update { currentState -> + currentState.copy( + pendingImagePlaceholderId = null, + selectedImagesByPlaceholderId = currentState.selectedImagesByPlaceholderId + + (placeholderId to SelectedImportedImage(importedDrawable.resourceName, importedDrawable.drawableReference)) + ) + } + _uiEffect.send(ComputerVisionEffect.ShowToast(R.string.msg_placeholder_image_selected)) + }.onFailure { exception -> + _uiState.update { it.copy(pendingImagePlaceholderId = null) } + _uiEffect.send(ComputerVisionEffect.ShowError("Image import failed: ${exception.message}")) + } + } + } + + private fun removePlaceholderImage(placeholderId: String) { + val state = _uiState.value + val importedImageInfo = state.selectedImagesByPlaceholderId[placeholderId] ?: return + + viewModelScope.launch { + removePlaceholderImageUC(state.layoutFilePath, importedImageInfo.resourceName) + .onSuccess { + _uiState.update { currentState -> + currentState.copy(selectedImagesByPlaceholderId = currentState.selectedImagesByPlaceholderId - placeholderId) + } + _uiEffect.send(ComputerVisionEffect.ShowToast(R.string.msg_image_removed)) + }.onFailure { exception -> + _uiEffect.send(ComputerVisionEffect.ShowError("Failed to clean up image file: ${exception.message}")) + } + } + } + + private fun resolvePlaceholderId(detection: DetectionResult): String { + val index = _uiState.value.detections.getSortedPlaceholders().indexOf(detection) + return "ph_${index.coerceAtLeast(0)}" + } + + fun isImagePlaceholderAt(imageX: Float, imageY: Float): Boolean { + return findImagePlaceholderAt(imageX, imageY) != null + } + + private fun findImagePlaceholderAt(imageX: Float, imageY: Float): DetectionResult? { + return _uiState.value.detections + .getSortedPlaceholders() + .firstOrNull { it.boundingBox.contains(imageX, imageY) } + } + + override fun onCleared() { + super.onCleared() + repository.release() + } + + companion object { + private const val TAG = "ComputerVisionViewModel" + + /** Standard Android phone viewport in dp used as the XML layout target size. */ + private const val TARGET_DP_WIDTH = 360 + private const val TARGET_DP_HEIGHT = 640 + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/BitmapUtils.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/BitmapUtils.kt new file mode 100644 index 0000000..2f34757 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/BitmapUtils.kt @@ -0,0 +1,167 @@ +package org.appdevforall.codeonthego.computervision.utils + +import android.graphics.Bitmap +import android.graphics.Color +import android.graphics.RectF +import kotlin.math.abs +import kotlin.math.exp +import kotlin.math.roundToInt + +object BitmapUtils { + + private const val EDGE_DETECTION_THRESHOLD = 30 + + fun preprocessForOcr(bitmap: Bitmap, blockSize: Int = 31, c: Int = 15): Bitmap { + val width = bitmap.width + val height = bitmap.height + val pixels = IntArray(width * height) + bitmap.getPixels(pixels, 0, width, 0, 0, width, height) + + val gray = toGrayscale(pixels) + normalize(gray) + val blurred = gaussianBlur(gray, width, height, blockSize) + val binary = adaptiveThreshold(gray, blurred, width, height, c) + medianFilter(binary, width, height) + + val outputPixels = IntArray(width * height) + for (i in binary.indices) { + outputPixels[i] = if (binary[i] > 0) Color.WHITE else Color.BLACK + } + return Bitmap.createBitmap(outputPixels, width, height, Bitmap.Config.ARGB_8888) + } + + fun cropRegion(bitmap: Bitmap, rect: RectF, padding: Int = 0): Bitmap { + val left = maxOf(0, (rect.left - padding).toInt()) + val top = maxOf(0, (rect.top - padding).toInt()) + val right = minOf(bitmap.width, (rect.right + padding).toInt()) + val bottom = minOf(bitmap.height, (rect.bottom + padding).toInt()) + val w = right - left + val h = bottom - top + if (w <= 0 || h <= 0) return bitmap + return Bitmap.createBitmap(bitmap, left, top, w, h) + } + + fun calculateVerticalProjection(bitmap: Bitmap): FloatArray { + val width = bitmap.width + val height = bitmap.height + val pixels = IntArray(width * height) + bitmap.getPixels(pixels, 0, width, 0, 0, width, height) + + val projection = FloatArray(width) + if (width < 3 || height == 0) { + return projection + } + + for (y in 0 until height) { + val rowOffset = y * width + for (x in 1 until width - 1) { + val leftPixel = pixels[rowOffset + x - 1] + val rightPixel = pixels[rowOffset + x + 1] + + val rLeft = (leftPixel shr 16) and 0xFF + val rRight = (rightPixel shr 16) and 0xFF + + val diff = abs(rLeft - rRight) + if (diff > EDGE_DETECTION_THRESHOLD) { + projection[x] += 1f + } + } + } + return projection + } + + private fun toGrayscale(pixels: IntArray): IntArray { + val gray = IntArray(pixels.size) + for (i in pixels.indices) { + val p = pixels[i] + val r = (p shr 16) and 0xFF + val g = (p shr 8) and 0xFF + val b = p and 0xFF + gray[i] = (0.299 * r + 0.587 * g + 0.114 * b).toInt() + } + return gray + } + + private fun normalize(gray: IntArray) { + var min = 255 + var max = 0 + for (v in gray) { + if (v < min) min = v + if (v > max) max = v + } + val range = max - min + if (range == 0) return + for (i in gray.indices) { + gray[i] = ((gray[i] - min) * 255) / range + } + } + + private fun gaussianBlur(gray: IntArray, width: Int, height: Int, blockSize: Int): IntArray { + val sigma = 0.3 * ((blockSize - 1) * 0.5 - 1) + 0.8 + val halfKernel = blockSize / 2 + val kernel = FloatArray(blockSize) + var kernelSum = 0.0f + for (i in 0 until blockSize) { + val x = i - halfKernel + kernel[i] = exp(-(x * x) / (2.0 * sigma * sigma)).toFloat() + kernelSum += kernel[i] + } + for (i in kernel.indices) kernel[i] /= kernelSum + + val temp = IntArray(width * height) + for (y in 0 until height) { + for (x in 0 until width) { + var sum = 0.0f + for (k in 0 until blockSize) { + val sx = (x + k - halfKernel).coerceIn(0, width - 1) + sum += gray[y * width + sx] * kernel[k] + } + temp[y * width + x] = sum.roundToInt() + } + } + + val blurred = IntArray(width * height) + for (x in 0 until width) { + for (y in 0 until height) { + var sum = 0.0f + for (k in 0 until blockSize) { + val sy = (y + k - halfKernel).coerceIn(0, height - 1) + sum += temp[sy * width + x] * kernel[k] + } + blurred[y * width + x] = sum.roundToInt() + } + } + return blurred + } + + private fun adaptiveThreshold( + gray: IntArray, + blurred: IntArray, + width: Int, + height: Int, + c: Int + ): IntArray { + val binary = IntArray(width * height) + for (i in gray.indices) { + binary[i] = if (gray[i] > blurred[i] - c) 255 else 0 + } + return binary + } + + private fun medianFilter(pixels: IntArray, width: Int, height: Int) { + val copy = pixels.copyOf() + val window = IntArray(9) + for (y in 1 until height - 1) { + for (x in 1 until width - 1) { + var idx = 0 + for (dy in -1..1) { + for (dx in -1..1) { + window[idx++] = copy[(y + dy) * width + (x + dx)] + } + } + window.sort() + pixels[y * width + x] = window[4] + } + } + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/CvAnalyticsUtils.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/CvAnalyticsUtils.kt new file mode 100644 index 0000000..d902399 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/CvAnalyticsUtils.kt @@ -0,0 +1,69 @@ +package org.appdevforall.codeonthego.computervision.utils + +import android.os.Bundle +import android.util.Log +import org.appdevforall.codeonthego.computervision.utils.CvAnalyticsUtil.EventNames.CV_DETECTION_COMPLETED +import org.appdevforall.codeonthego.computervision.utils.CvAnalyticsUtil.EventNames.CV_DETECTION_STARTED +import org.appdevforall.codeonthego.computervision.utils.CvAnalyticsUtil.EventNames.CV_IMAGE_SELECTED +import org.appdevforall.codeonthego.computervision.utils.CvAnalyticsUtil.EventNames.CV_SCREEN_OPENED +import org.appdevforall.codeonthego.computervision.utils.CvAnalyticsUtil.EventNames.CV_XML_EXPORTED +import org.appdevforall.codeonthego.computervision.utils.CvAnalyticsUtil.EventNames.CV_XML_GENERATED + +object CvAnalyticsUtil { + private const val TAG = "CvAnalyticsUtil" + + private fun logEvent(eventName: String, params: Bundle) { + Log.d(TAG, "Analytics event skipped in standalone plugin: $eventName $params") + } + + fun trackScreenOpened() { + logEvent(CV_SCREEN_OPENED, Bundle().apply { + putLong("timestamp", System.currentTimeMillis()) + }) + } + + fun trackImageSelected(fromCamera: Boolean) { + logEvent(CV_IMAGE_SELECTED, Bundle().apply { + putString("source", if (fromCamera) "camera" else "gallery") + putLong("timestamp", System.currentTimeMillis()) + }) + } + + fun trackDetectionStarted() { + logEvent(CV_DETECTION_STARTED, Bundle().apply { + putLong("timestamp", System.currentTimeMillis()) + }) + } + + fun trackDetectionCompleted(success: Boolean, detectionCount: Int, durationMs: Long) { + logEvent(CV_DETECTION_COMPLETED, Bundle().apply { + putBoolean("success", success) + putInt("detection_count", detectionCount) + putLong("duration_ms", durationMs) + putLong("timestamp", System.currentTimeMillis()) + }) + } + + fun trackXmlGenerated(componentCount: Int) { + logEvent(CV_XML_GENERATED, Bundle().apply { + putInt("component_count", componentCount) + putLong("timestamp", System.currentTimeMillis()) + }) + } + + fun trackXmlExported(toDownloads: Boolean) { + logEvent(CV_XML_EXPORTED, Bundle().apply { + putString("export_method", if (toDownloads) "save_downloads" else "update_layout") + putLong("timestamp", System.currentTimeMillis()) + }) + } + + private object EventNames { + const val CV_SCREEN_OPENED = "cv_screen_opened" + const val CV_IMAGE_SELECTED = "cv_image_selected" + const val CV_DETECTION_STARTED = "cv_detection_started" + const val CV_DETECTION_COMPLETED = "cv_detection_completed" + const val CV_XML_GENERATED = "cv_xml_generated" + const val CV_XML_EXPORTED = "cv_xml_exported" + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/DetectionVisualizer.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/DetectionVisualizer.kt new file mode 100644 index 0000000..560c343 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/DetectionVisualizer.kt @@ -0,0 +1,227 @@ +package org.appdevforall.codeonthego.computervision.utils + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint +import android.graphics.RectF +import android.graphics.drawable.Drawable +import androidx.appcompat.content.res.AppCompatResources +import androidx.core.graphics.toColorInt +import com.appdevforall.sketchtoui.plugin.R +import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult + + +/** + * Utility class responsible for visualizing computer vision detection results. + * It handles drawing bounding boxes, text labels, and interactive visual hints + * directly onto image bitmaps. + * + * @property context The context used to retrieve resources such as drawables. + */ +class DetectionVisualizer(private val context: Context) { + + private val boundingBoxPaint by lazy { + Paint().apply { + color = Color.GREEN + style = Paint.Style.STROKE + strokeWidth = 5.0f + alpha = 200 + } + } + + private val imagePlaceholderPaint by lazy { + Paint().apply { + color = "#FF8A00".toColorInt() + style = Paint.Style.STROKE + strokeWidth = 7.0f + alpha = 230 + } + } + + private val imagePlaceholderFillPaint by lazy { + Paint().apply { + color = "#FF8A00".toColorInt() + style = Paint.Style.FILL + alpha = 40 + } + } + + private val textRecognitionBoxPaint by lazy { + Paint().apply { + color = Color.BLUE + style = Paint.Style.STROKE + strokeWidth = 3.0f + alpha = 200 + } + } + + private val textPaint by lazy { + Paint().apply { + color = Color.WHITE + style = Paint.Style.FILL + textSize = 40.0f + setShadowLayer(5.0f, 0f, 0f, Color.BLACK) + } + } + + private val imagePlaceholderUploadDrawable: Drawable? by lazy { + AppCompatResources.getDrawable(context, R.drawable.ic_placeholder_upload)?.mutate()?.apply { + setTint(Color.WHITE) + } + } + + private val imagePlaceholderDeleteDrawable: Drawable? by lazy { + AppCompatResources.getDrawable(context, R.drawable.ic_placeholder_delete)?.mutate()?.apply { + setTint(Color.WHITE) + } + } + + private val imagePlaceholderBadgePaint by lazy { + Paint(Paint.ANTI_ALIAS_FLAG).apply { + color = "#CC111111".toColorInt() + style = Paint.Style.FILL + } + } + + private val deleteBadgeBackgroundPaint by lazy { + Paint(Paint.ANTI_ALIAS_FLAG).apply { + color = "#CCB3261E".toColorInt() + style = Paint.Style.FILL + } + } + + private val deleteIconClickableAreas = mutableMapOf() + + /** + * Draws detection bounding boxes, labels, and interactive placeholder hints on a given bitmap. + * + * @param bitmap The original image on which detections were performed. + * @param detections A list of [DetectionResult] objects containing the bounding boxes and labels. + * @param selectedPlaceholderIds A set of IDs representing image placeholders that have been selected/filled. + * @return A new [Bitmap] instance with the visualized detections drawn over it. + */ + fun visualize( + bitmap: Bitmap, + detections: List, + selectedPlaceholderIds: Set + ): Bitmap { + val mutableBitmap = bitmap.copy(Bitmap.Config.ARGB_8888, true) + val canvas = Canvas(mutableBitmap) + + deleteIconClickableAreas.clear() + + val placeholderIdsByDetection = mapPlaceholderDetections(detections) + + for (result in detections) { + if (result.label == IMAGE_PLACEHOLDER_LABEL) { + val placeholderId = placeholderIdsByDetection[result] ?: continue + val hasSelectedImage = selectedPlaceholderIds.contains(placeholderId) + + drawImagePlaceholderHint(canvas, result.boundingBox, hasSelectedImage, placeholderId) + } else { + drawStandardDetection(canvas, result) + } + } + + return mutableBitmap + } + + /** + * Checks if the given X, Y coordinates intersect with any drawn delete icon. + * @return The placeholderId if a delete icon was tapped, null otherwise. + */ + fun getTappedDeleteIconId(x: Float, y: Float): String? { + return deleteIconClickableAreas.entries.firstOrNull { it.value.contains(x, y) }?.key + } + + /** + * Filters and maps image placeholder detections to their corresponding auto-generated IDs. + * Placeholders are sorted from top to bottom, left to right to ensure consistent ID assignment. + * + * @param detections The full list of detections. + * @return A map linking each placeholder [DetectionResult] to its generated string ID (e.g., "ph_0"). + */ + private fun mapPlaceholderDetections(detections: List): Map { + return detections + .filter { it.label == IMAGE_PLACEHOLDER_LABEL } + .sortedWith(compareBy({ it.boundingBox.top }, { it.boundingBox.left })) + .mapIndexed { index, detection -> + detection to "ph_$index" + }.toMap() + } + + /** + * Draws a standard detection bounding box and its corresponding text label. + * It uses different colors depending on whether the detection is from YOLO or Text Recognition. + * + * @param canvas The canvas to draw the detection on. + * @param result The [DetectionResult] containing the coordinates, label, and detection type. + */ + private fun drawStandardDetection(canvas: Canvas, result: DetectionResult) { + val paint = if (result.isYolo) boundingBoxPaint else textRecognitionBoxPaint + canvas.drawRect(result.boundingBox, paint) + + val label = result.label.take(15) + val text = if (result.text.isNotEmpty()) "$label: ${result.text}" else label + canvas.drawText(text, result.boundingBox.left, result.boundingBox.top - 5, textPaint) + } + + /** + * Draws a highlighted region and a central badge for an image placeholder detection. + * + * @param canvas The canvas to draw the hint on. + * @param boundingBox The rectangular bounds of the detected image placeholder. + * @param hasSelectedImage A flag indicating whether the user has already selected an image for this placeholder. + */ + private fun drawImagePlaceholderHint( + canvas: Canvas, + boundingBox: RectF, + hasSelectedImage: Boolean, + placeholderId: String + ) { + canvas.drawRect(boundingBox, imagePlaceholderFillPaint) + canvas.drawRect(boundingBox, imagePlaceholderPaint) + + val badgeHeight = (boundingBox.height() * 0.24f).coerceIn(44f, 72f) + val badgeTop = (boundingBox.centerY() - badgeHeight / 2f).coerceAtLeast(boundingBox.top + 8f) + val badgeBottom = (badgeTop + badgeHeight).coerceAtMost(boundingBox.bottom - 8f) + + val badgeRect = RectF( + boundingBox.left + 12f, + badgeTop, + boundingBox.right - 12f, + badgeBottom + ) + + val bgPaint = if (hasSelectedImage) deleteBadgeBackgroundPaint else imagePlaceholderBadgePaint + canvas.drawRoundRect(badgeRect, 16f, 16f, bgPaint) + + drawPlaceholderIcon(canvas, badgeRect, hasSelectedImage) + + if (hasSelectedImage) { + deleteIconClickableAreas[placeholderId] = badgeRect + } + } + + private fun drawPlaceholderIcon(canvas: Canvas, badgeRect: RectF, hasSelectedImage: Boolean) { + val iconDrawable = if (hasSelectedImage) imagePlaceholderDeleteDrawable else imagePlaceholderUploadDrawable + if (iconDrawable == null) return + + val iconSize = minOf(badgeRect.width(), badgeRect.height()) * 0.80f + + val left = (badgeRect.centerX() - iconSize / 2f).toInt() + val top = (badgeRect.centerY() - iconSize / 2f).toInt() + val right = (badgeRect.centerX() + iconSize / 2f).toInt() + val bottom = (badgeRect.centerY() + iconSize / 2f).toInt() + + iconDrawable.alpha = 255 + iconDrawable.setBounds(left, top, right, bottom) + iconDrawable.draw(canvas) + } + + fun clearCache() { + deleteIconClickableAreas.clear() + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/MetadataDetector.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/MetadataDetector.kt new file mode 100644 index 0000000..68cee40 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/MetadataDetector.kt @@ -0,0 +1,51 @@ +package org.appdevforall.codeonthego.computervision.utils + +object MetadataDetector { + private val metadataSnippets = listOf( + "" + ) + + private val metadataKeywords = listOf( + "layout_width", + "layout_height", + "layout_margin", + "layout_gravity", + "textstyle", + "textcolor", + "textsize", + "padding", + "orientation", + "baselinealigned", + "match_parent", + "wrap_content" + ) + + private val xmlAttributeRegex = Regex("""\b(?:android|app|tools):[a-zA-Z_]+\b""") + fun isCanvasMetadata(text: String): Boolean { + val lowerText = text.lowercase() + if (lowerText.isBlank()) return false + if (metadataSnippets.any { snippet -> lowerText.contains(snippet) }) return true + if (xmlAttributeRegex.containsMatchIn(lowerText)) return true + if (metadataKeywords.any { keyword -> lowerText.contains(keyword) }) return true + return false + } + + fun isMetadataLabel(label: String): Boolean { + val normalized = label.trim().lowercase() + return normalized == "margin_metadata" || normalized.contains("metadata") + } + + fun isMetadataDetection(label: String, text: String): Boolean { + return isMetadataLabel(label) || isCanvasMetadata(text) + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/OcrExtensions.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/OcrExtensions.kt new file mode 100644 index 0000000..b0030dd --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/OcrExtensions.kt @@ -0,0 +1,29 @@ +package org.appdevforall.codeonthego.computervision.utils + +private val PUNCTUATION_DELIMITERS = Regex("\\s*[,;|/\\n]+\\s*") +private val WHITESPACE_DELIMITERS = Regex("\\s+") +private val OCR_NUMERIC_PATTERN = Regex("^[0-9oOlIzZsSbB]+$") + +/** + * Extracts separated entries from a raw OCR string. + * Tries to split by punctuation first. If no punctuation is found, + * it falls back to space-separated tokens, provided they all look like numbers or OCR artifacts. + */ +internal fun String.extractOcrEntries(): List { + // Try to split by explicit punctuation or newlines + val punctuatedTokens = this.split(PUNCTUATION_DELIMITERS).filter { it.isNotBlank() } + + // If successfully split into multiple items (or none), return them + if (punctuatedTokens.size != 1) { + return punctuatedTokens + } + + // Fallback: check if elements were separated only by spaces + val whitespaceTokens = this.trim().split(WHITESPACE_DELIMITERS).filter { it.isNotBlank() } + + // Only accept space separation if all resulting tokens look like numbers (or OCR artifacts) + val isSpaceSeparatedOcrNumbers = whitespaceTokens.size > 1 && + whitespaceTokens.all { it.matches(OCR_NUMERIC_PATTERN) } + + return if (isSpaceSeparatedOcrNumbers) whitespaceTokens else punctuatedTokens +} \ No newline at end of file diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/OcrTextAssembler.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/OcrTextAssembler.kt new file mode 100644 index 0000000..21dec61 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/OcrTextAssembler.kt @@ -0,0 +1,44 @@ +package org.appdevforall.codeonthego.computervision.utils + +import com.google.mlkit.vision.text.Text + +object OcrTextAssembler { + const val DEFAULT_SPACE_GAP_TOLERANCE_PX = 15f + + fun extractTextWithTolerance( + textBlocks: List, + maxSpaceGap: Float = DEFAULT_SPACE_GAP_TOLERANCE_PX + ): String { + return textBlocks.joinToString(" ") { block -> + block.lines.joinToString(" ") { line -> + joinElementsWithTolerance(line, maxSpaceGap) + } + }.trim() + } + + fun joinElementsWithTolerance(line: Text.Line, maxSpaceGap: Float = DEFAULT_SPACE_GAP_TOLERANCE_PX): String { + val elements = line.elements.sortedBy { it.boundingBox?.left ?: 0 } + if (elements.isEmpty()) return "" + + val builder = StringBuilder() + var prevRight = elements.first().boundingBox?.right ?: 0 + builder.append(elements.first().text) + + for (i in 1 until elements.size) { + val current = elements[i] + val currentBox = current.boundingBox + + if (currentBox != null) { + val gap = currentBox.left - prevRight + if (gap > maxSpaceGap) { + builder.append(" ") + } + prevRight = currentBox.right + } else { + builder.append(" ") + } + builder.append(current.text) + } + return builder.toString().trim() + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/PlaceholderUtils.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/PlaceholderUtils.kt new file mode 100644 index 0000000..1a2eff8 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/PlaceholderUtils.kt @@ -0,0 +1,30 @@ +package org.appdevforall.codeonthego.computervision.utils + +import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult +import org.appdevforall.codeonthego.computervision.domain.model.ScaledBox + +const val IMAGE_PLACEHOLDER_LABEL = "image_placeholder" + +fun List.getSortedPlaceholders(): List { + return this.filter { it.label == IMAGE_PLACEHOLDER_LABEL } + .sortedWith(compareBy({ it.boundingBox.top }, { it.boundingBox.left })) +} + +fun List.getSortedScaledPlaceholders(): List { + return this.filter { it.label == IMAGE_PLACEHOLDER_LABEL } + .sortedWith(compareBy({ it.y }, { it.x })) +} + +/** + * Associates ordered image placeholders with their selected drawable references. + * Useful for mapping user-selected gallery images to the physical canvas bounding boxes. + */ +fun List.buildPlaceholderOverrides(selectedImagesByPlaceholderId: Map): Map { + val placeholders = this.getSortedScaledPlaceholders() + + return placeholders.mapIndexedNotNull { index, box -> + val drawableReference = selectedImagesByPlaceholderId["ph_$index"] + ?: return@mapIndexedNotNull null + box to drawableReference + }.toMap() +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/SmartBoundaryDetector.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/SmartBoundaryDetector.kt new file mode 100644 index 0000000..a72ab07 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/SmartBoundaryDetector.kt @@ -0,0 +1,103 @@ +package org.appdevforall.codeonthego.computervision.utils + +import android.graphics.Bitmap +import org.appdevforall.codeonthego.computervision.utils.BitmapUtils.calculateVerticalProjection + +object SmartBoundaryDetector { + + private const val DEFAULT_EDGE_IGNORE_PERCENT = 0.05f + private const val LEFT_ZONE_END_PERCENT = 0.5f + private const val RIGHT_ZONE_START_PERCENT = 0.5f + private const val MIN_GAP_WIDTH_PERCENT = 0.02 + private const val PRIMARY_ACTIVITY_THRESHOLD = 0.05f + private const val FALLBACK_ACTIVITY_THRESHOLD = 0.01f + private const val LEFT_FALLBACK_BOUND_PERCENT = 0.15f + private const val RIGHT_FALLBACK_BOUND_PERCENT = 0.85f + + fun detectSmartBoundaries( + bitmap: Bitmap, + edgeIgnorePercent: Float = DEFAULT_EDGE_IGNORE_PERCENT + ): Pair { + val width = bitmap.width + val projection = calculateVerticalProjection(bitmap) + val minimumGapWidth = (width * MIN_GAP_WIDTH_PERCENT).toInt() + + val ignoredEdgePixels = (width * edgeIgnorePercent).toInt() + val leftZoneEnd = (width * LEFT_ZONE_END_PERCENT).toInt() + val rightZoneStart = (width * RIGHT_ZONE_START_PERCENT).toInt() + val rightZoneEnd = width - ignoredEdgePixels + + if (ignoredEdgePixels >= leftZoneEnd || rightZoneStart >= rightZoneEnd) { + return Pair( + (width * LEFT_FALLBACK_BOUND_PERCENT).toInt(), + (width * RIGHT_FALLBACK_BOUND_PERCENT).toInt() + ) + } + + val leftSignal = projection.copyOfRange(ignoredEdgePixels, leftZoneEnd) + var (leftBound, leftGapLength) = findBestGapMidpoint(leftSignal, offset = ignoredEdgePixels) + if (leftBound == null || leftGapLength < minimumGapWidth) { + leftBound = findBestGapMidpoint(leftSignal, offset = ignoredEdgePixels, normalizeSignal = true).first + } + + val rightSignal = projection.copyOfRange(rightZoneStart, rightZoneEnd) + var (rightBound, rightGapLength) = findBestGapMidpoint(rightSignal, offset = rightZoneStart) + if (rightBound == null || rightGapLength < minimumGapWidth) { + rightBound = findBestGapMidpoint(rightSignal, offset = rightZoneStart, normalizeSignal = true).first + } + + val finalLeftBound = leftBound ?: (width * LEFT_FALLBACK_BOUND_PERCENT).toInt() + val finalRightBound = rightBound ?: (width * RIGHT_FALLBACK_BOUND_PERCENT).toInt() + return Pair(finalLeftBound, finalRightBound) + } + + private fun findBestGapMidpoint( + signalSegment: FloatArray, + offset: Int = 0, + normalizeSignal: Boolean = false + ): Pair { + if (signalSegment.isEmpty()) { + return Pair(null, 0) + } + + val signal = if (normalizeSignal) { + val minValue = signalSegment.minOrNull() ?: 0f + FloatArray(signalSegment.size) { index -> signalSegment[index] - minValue } + } else { + signalSegment + } + + val activityThresholdMultiplier = if (normalizeSignal) { + FALLBACK_ACTIVITY_THRESHOLD + } else { + PRIMARY_ACTIVITY_THRESHOLD + } + val threshold = (signal.maxOrNull() ?: 0f) * activityThresholdMultiplier + + var maxGapLength = 0 + var maxGapMidpoint: Int? = null + var currentGapStart = -1 + var previousIsActive = true + + signal.forEachIndexed { index, value -> + val isActive = value > threshold + if (previousIsActive && !isActive) { + currentGapStart = index + } + + val isGapClosing = currentGapStart != -1 && (index + 1 == signal.size || (!isActive && signal[index + 1] > threshold)) + if (isGapClosing) { + val gapLength = index - currentGapStart + 1 + if (gapLength > maxGapLength) { + maxGapLength = gapLength + maxGapMidpoint = currentGapStart + (gapLength / 2) + } + currentGapStart = -1 + } + + previousIsActive = isActive + } + + return Pair(maxGapMidpoint?.plus(offset), maxGapLength) + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/TextCleaner.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/TextCleaner.kt new file mode 100644 index 0000000..495744a --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/TextCleaner.kt @@ -0,0 +1,35 @@ +package org.appdevforall.codeonthego.computervision.utils + +object TextCleaner { + + private val nonAlphanumericRegex = Regex("[^a-zA-Z0-9 ]") + private val leadingMarkerRegex = Regex("^[\\[\\]()●○□☑✓-]+\\s*") + private val leadingStandaloneCircleRegex = Regex("^[O0o]\\s+") + private val duplicatedLeadingCircleRegex = Regex("^[O0o](?=[oO][a-z])") + + fun cleanText(text: String): String { + return text.replace("\n", " ") + .replace(nonAlphanumericRegex, "") + .trim() + } + + fun cleanTextStrippingLeadingO(text: String): String { + val cleanedText = text.trim() + .replace(leadingMarkerRegex, "") + .replace(leadingStandaloneCircleRegex, "") + .replace(duplicatedLeadingCircleRegex, "") + + return cleanedText.ifEmpty { text } + } + + fun cleanTextPreservingLeadingO(text: String): String { + var cleanedText = text.trim() + .replace(Regex("^[\\[\\]()●○□☑✓-]+\\s*"), "") + + cleanedText = cleanedText.replace(Regex("^[DT]?opti[oa]n", RegexOption.IGNORE_CASE), "Option") + cleanedText = cleanedText.replace(Regex("^pti[oa]n", RegexOption.IGNORE_CASE), "Option") + cleanedText = cleanedText.replace(Regex("^optton", RegexOption.IGNORE_CASE), "Option") + + return cleanedText.ifEmpty { text } + } +} diff --git a/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/XmlFileManager.kt b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/XmlFileManager.kt new file mode 100644 index 0000000..4cd00c5 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/java/org/appdevforall/codeonthego/computervision/utils/XmlFileManager.kt @@ -0,0 +1,90 @@ +package org.appdevforall.codeonthego.computervision.utils + +import android.content.ContentValues +import android.content.Context +import android.os.Build +import android.os.Environment +import android.provider.MediaStore +import androidx.annotation.RequiresApi +import java.io.File +import java.io.FileOutputStream +import java.io.IOException + +/** + * Utility class responsible for managing file input/output operations. + * Specifically handles saving XML files to the device's public Downloads directory, + * adapting to different Android API levels (Scoped Storage vs Legacy). + * + * @property context The application or activity context needed to access content resolvers. + */ +class XmlFileManager(private val context: Context) { + + /** + * Saves an XML string to a file in the device's public Downloads directory. + * It automatically handles the differences in file storage APIs between + * Android 10 (API 29+) and older versions. + * + * @param xmlString The XML content to be saved. + * @param fileName The desired name for the output file. Defaults to "layout_result.xml". + * @return A [Result] containing the file name if successful, or an [IOException] on failure. + */ + fun saveXmlToDownloads(xmlString: String, fileName: String = "layout_result.xml"): Result { + return try { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + saveUsingMediaStore(xmlString, fileName) + } else { + saveUsingLegacyFileApi(xmlString, fileName) + } + Result.success(fileName) + } catch (e: IOException) { + Result.failure(e) + } + } + + /** + * Saves the file using the MediaStore API. + * This is the required approach for Android 10 (API 29) and above due to Scoped Storage restrictions. + * + * @param xmlString The XML content to write. + * @param fileName The name of the file. + * @throws IOException If the MediaStore record cannot be created or written to. + */ + @RequiresApi(Build.VERSION_CODES.Q) + private fun saveUsingMediaStore(xmlString: String, fileName: String) { + val resolver = context.contentResolver + val contentValues = ContentValues().apply { + put(MediaStore.MediaColumns.DISPLAY_NAME, fileName) + put(MediaStore.MediaColumns.MIME_TYPE, "text/xml") + put(MediaStore.MediaColumns.RELATIVE_PATH, Environment.DIRECTORY_DOWNLOADS) + } + + val uri = resolver.insert(MediaStore.Downloads.EXTERNAL_CONTENT_URI, contentValues) + ?: throw IOException("Failed to create new MediaStore record for $fileName.") + + resolver.openOutputStream(uri)?.use { outputStream -> + outputStream.write(xmlString.toByteArray()) + } ?: throw IOException("Failed to open output stream for MediaStore URI.") + } + + /** + * Saves the file using the legacy File API. + * This approach is used for devices running Android 9 (API 28) or lower. + * + * @param xmlString The XML content to write. + * @param fileName The name of the file. + * @throws IOException If the Downloads directory or the file cannot be created. + */ + @Suppress("DEPRECATION") + private fun saveUsingLegacyFileApi(xmlString: String, fileName: String) { + val downloadsDir = Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS) + + if (!downloadsDir.exists() && !downloadsDir.mkdirs()) { + throw IOException("Failed to create Downloads directory.") + } + + val file = File(downloadsDir, fileName) + FileOutputStream(file).use { outputStream -> + outputStream.write(xmlString.toByteArray()) + } + } +} diff --git a/sketch-to-ui-plugin/src/main/kotlin/com/appdevforall/sketchtoui/plugin/GeneratedStringsWriter.kt b/sketch-to-ui-plugin/src/main/kotlin/com/appdevforall/sketchtoui/plugin/GeneratedStringsWriter.kt new file mode 100644 index 0000000..2054c14 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/kotlin/com/appdevforall/sketchtoui/plugin/GeneratedStringsWriter.kt @@ -0,0 +1,34 @@ +package com.appdevforall.sketchtoui.plugin + +import com.itsaky.androidide.plugins.services.IdeFileService +import com.itsaky.androidide.plugins.services.IdeProjectService +import java.io.File + +class GeneratedStringsWriter( + private val projectService: IdeProjectService?, + private val fileService: IdeFileService? +) { + fun write(stringsXml: String?): Boolean { + if (stringsXml.isNullOrBlank()) return true + val service = fileService ?: return false + val projectRoot = projectService?.getCurrentProject()?.rootDir ?: return false + val stringsFile = File(projectRoot, STRINGS_XML_RELATIVE_PATH) + val existing = service.readFile(stringsFile) + + val updated = if (existing.isNullOrBlank()) { + "\n${stringsXml.trim().prependIndent(" ")}\n\n" + } else { + val insertionPoint = existing.lastIndexOf("") + if (insertionPoint < 0) return false + existing.take(insertionPoint).trimEnd() + + "\n${stringsXml.trim().prependIndent(" ")}\n" + + existing.substring(insertionPoint) + } + + return service.writeFile(stringsFile, updated) + } + + private companion object { + const val STRINGS_XML_RELATIVE_PATH = "app/src/main/res/values/strings.xml" + } +} diff --git a/sketch-to-ui-plugin/src/main/kotlin/com/appdevforall/sketchtoui/plugin/SketchToUiPlugin.kt b/sketch-to-ui-plugin/src/main/kotlin/com/appdevforall/sketchtoui/plugin/SketchToUiPlugin.kt new file mode 100644 index 0000000..a8e0cac --- /dev/null +++ b/sketch-to-ui-plugin/src/main/kotlin/com/appdevforall/sketchtoui/plugin/SketchToUiPlugin.kt @@ -0,0 +1,133 @@ +package com.appdevforall.sketchtoui.plugin + +import android.content.Context +import android.widget.Toast +import com.appdevforall.sketchtoui.plugin.fragments.SketchToUiFragment +import com.itsaky.androidide.plugins.IPlugin +import com.itsaky.androidide.plugins.PluginContext +import com.itsaky.androidide.plugins.extensions.DocumentationExtension +import com.itsaky.androidide.plugins.extensions.MenuItem +import com.itsaky.androidide.plugins.extensions.NavigationItem +import com.itsaky.androidide.plugins.extensions.PluginTooltipEntry +import com.itsaky.androidide.plugins.extensions.ToolbarAction +import com.itsaky.androidide.plugins.extensions.UIExtension +import com.itsaky.androidide.plugins.services.IdeEditorService +import com.itsaky.androidide.plugins.services.IdeUIService +import java.io.File + +class SketchToUiPlugin : IPlugin, UIExtension, DocumentationExtension { + + private lateinit var context: PluginContext + + override fun initialize(context: PluginContext): Boolean { + this.context = context + pluginAndroidContext = context.androidContext + context.logger.info("SketchToUiPlugin initialized") + return true + } + + override fun activate(): Boolean = true + + override fun deactivate(): Boolean = true + + override fun dispose() = Unit + + override fun getMainMenuItems(): List { + return listOf( + MenuItem( + id = "sketch_to_ui", + title = "Sketch to UI", + isEnabled = true, + isVisible = true, + shortcut = null, + subItems = emptyList(), + action = { openSketchToUiIfValid() }, + tooltipTag = TOOLTIP_SKETCH_TO_UI, + icon = R.drawable.ic_computer_vision, + ).apply { + isEnabledProvider = { hasCurrentLayoutXml() } + } + ) + } + + override fun getSideMenuItems(): List = emptyList() + + override fun getToolbarActions(): List = emptyList() + + override fun getTooltipCategory(): String = "plugin_$PLUGIN_ID" + + override fun getTooltipEntries(): List = listOf( + PluginTooltipEntry( + tag = TOOLTIP_SKETCH_TO_UI, + summary = "Sketch to UI
Convert a layout sketch or screenshot into Android XML.", + detail = """ +

Sketch to UI

+

Use this action from an Android layout XML file to open the sketch conversion workflow.

+ +

Availability

+

The action is enabled only when the current editor file is a layout XML resource.

+ """.trimIndent() + ) + ) + + private fun openSketchToUiIfValid() { + val currentFile = getCurrentLayoutXml() + if (currentFile == null) { + context.logger.warn("Sketch to UI requires an Android layout XML file") + SketchToUiState.setLayoutFile(null, null) + showToast("Open an Android layout XML file before generating XML.") + return + } + + SketchToUiState.setLayoutFile(currentFile.absolutePath, currentFile.name) + openPluginScreen() + } + + private fun openPluginScreen() { + val opened = context.services + .get(IdeUIService::class.java) + ?.openPluginScreen( + pluginId = PLUGIN_ID, + fragmentClassName = SketchToUiFragment::class.java.name, + title = "Generate XML" + ) == true + + if (!opened) { + context.logger.warn("Unable to open Sketch to UI plugin screen") + } + } + + private fun hasCurrentLayoutXml(): Boolean { + val currentFile = context.services + .get(IdeEditorService::class.java) + ?.getCurrentFile() + ?: return false + + return currentFile.isLayoutXml() + } + + private fun getCurrentLayoutXml(): File? { + val currentFile = context.services + .get(IdeEditorService::class.java) + ?.getCurrentFile() + ?: return null + + return currentFile.takeIf { it.isLayoutXml() } + } + + private fun File.isLayoutXml(): Boolean = + extension.equals("xml", ignoreCase = true) && + parentFile?.name == "layout" + + private fun showToast(message: String) { + Toast.makeText(context.androidContext, message, Toast.LENGTH_SHORT).show() + } + + companion object { + const val PLUGIN_ID = "com.appdevforall.sketchtoui.plugin" + private const val TOOLTIP_SKETCH_TO_UI = "sketch_to_ui.main_feature" + private var pluginAndroidContext: Context? = null + + fun getPluginAndroidContext(): Context? = pluginAndroidContext + } +} diff --git a/sketch-to-ui-plugin/src/main/kotlin/com/appdevforall/sketchtoui/plugin/SketchToUiState.kt b/sketch-to-ui-plugin/src/main/kotlin/com/appdevforall/sketchtoui/plugin/SketchToUiState.kt new file mode 100644 index 0000000..3cc0ec7 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/kotlin/com/appdevforall/sketchtoui/plugin/SketchToUiState.kt @@ -0,0 +1,14 @@ +package com.appdevforall.sketchtoui.plugin + +object SketchToUiState { + @Volatile + var layoutFilePath: String? = null + + @Volatile + var layoutFileName: String? = null + + fun setLayoutFile(path: String?, name: String?) { + layoutFilePath = path + layoutFileName = name + } +} diff --git a/sketch-to-ui-plugin/src/main/kotlin/com/appdevforall/sketchtoui/plugin/fragments/SketchToUiFragment.kt b/sketch-to-ui-plugin/src/main/kotlin/com/appdevforall/sketchtoui/plugin/fragments/SketchToUiFragment.kt new file mode 100644 index 0000000..4b8048d --- /dev/null +++ b/sketch-to-ui-plugin/src/main/kotlin/com/appdevforall/sketchtoui/plugin/fragments/SketchToUiFragment.kt @@ -0,0 +1,559 @@ +package com.appdevforall.sketchtoui.plugin.fragments + +import android.app.Activity +import android.content.Intent +import android.graphics.Bitmap +import android.net.Uri +import android.os.Bundle +import android.util.Log +import android.view.LayoutInflater +import android.view.View +import android.view.ViewGroup +import android.widget.Button +import android.widget.Toast +import androidx.activity.OnBackPressedCallback +import androidx.activity.result.contract.ActivityResultContracts +import androidx.appcompat.app.AlertDialog +import androidx.fragment.app.Fragment +import androidx.lifecycle.lifecycleScope +import com.appdevforall.sketchtoui.plugin.GeneratedStringsWriter +import com.appdevforall.sketchtoui.plugin.R +import com.appdevforall.sketchtoui.plugin.SketchToUiPlugin +import com.appdevforall.sketchtoui.plugin.SketchToUiState +import com.google.android.material.appbar.MaterialToolbar +import com.google.android.material.color.MaterialColors +import com.itsaky.androidide.plugins.base.PluginFragmentHelper +import com.itsaky.androidide.plugins.services.IdeEditorService +import com.itsaky.androidide.plugins.services.IdeFileService +import com.itsaky.androidide.plugins.services.IdeProjectService +import com.itsaky.androidide.plugins.services.SelectionRange +import java.io.File +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import org.appdevforall.codeonthego.computervision.data.repository.VisionRepositoryImpl +import org.appdevforall.codeonthego.computervision.data.repository.DrawableImportHelper +import org.appdevforall.codeonthego.computervision.data.source.OcrSource +import org.appdevforall.codeonthego.computervision.data.source.YoloModelSource +import org.appdevforall.codeonthego.computervision.domain.GenericBoxResolver +import org.appdevforall.codeonthego.computervision.domain.RegionOcrProcessor +import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult +import org.appdevforall.codeonthego.computervision.domain.usecase.GenerateXmlUC +import org.appdevforall.codeonthego.computervision.domain.usecase.ImportPlaceholderImageUC +import org.appdevforall.codeonthego.computervision.domain.usecase.PrepareImageUC +import org.appdevforall.codeonthego.computervision.domain.usecase.RemovePlaceholderImageUC +import org.appdevforall.codeonthego.computervision.domain.usecase.RunVisionUC +import org.appdevforall.codeonthego.computervision.ui.CvOperation +import org.appdevforall.codeonthego.computervision.ui.GuidelinesView +import org.appdevforall.codeonthego.computervision.ui.SelectedImportedImage +import org.appdevforall.codeonthego.computervision.ui.ZoomableImageView +import org.appdevforall.codeonthego.computervision.utils.DetectionVisualizer +import org.appdevforall.codeonthego.computervision.utils.getSortedPlaceholders +import org.appdevforall.codeonthego.computervision.utils.XmlFileManager + +class SketchToUiFragment : Fragment() { + + private var selectedBitmap: Bitmap? = null + private var generatedXml: GeneratedXml? = null + private var currentDetections: List = emptyList() + private var parsedAnnotations: Map = emptyMap() + private var pendingImagePlaceholderId: String? = null + private var selectedImagesByPlaceholderId: Map = emptyMap() + private var leftGuidePct: Float = DEFAULT_LEFT_GUIDE_PCT + private var rightGuidePct: Float = DEFAULT_RIGHT_GUIDE_PCT + private val detectionVisualizer by lazy { DetectionVisualizer(requireContext()) } + private val drawableImportHelper by lazy { DrawableImportHelper(requireContext().contentResolver) } + private val importPlaceholderImageUC by lazy { ImportPlaceholderImageUC(drawableImportHelper) } + private val removePlaceholderImageUC by lazy { RemovePlaceholderImageUC(drawableImportHelper) } + + private lateinit var toolbar: MaterialToolbar + private lateinit var imageView: ZoomableImageView + private lateinit var guidelinesView: GuidelinesView + private lateinit var detectButton: Button + private lateinit var updateButton: Button + private lateinit var saveButton: Button + + private val pickImageLauncher = registerForActivityResult( + ActivityResultContracts.StartActivityForResult() + ) { result -> + if (result.resultCode == Activity.RESULT_OK) { + result.data?.data?.let(::loadImage) + } + } + + private val pickPlaceholderImageLauncher = registerForActivityResult( + ActivityResultContracts.GetContent() + ) { uri -> + uri?.let(::loadPlaceholderImage) + ?: toast(getString(R.string.msg_no_image_selected)) + } + + override fun onGetLayoutInflater(savedInstanceState: Bundle?): LayoutInflater { + val inflater = super.onGetLayoutInflater(savedInstanceState) + return PluginFragmentHelper.getPluginInflater(SketchToUiPlugin.PLUGIN_ID, inflater) + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + requireActivity().onBackPressedDispatcher.addCallback( + this, + object : OnBackPressedCallback(false) { + override fun handleOnBackPressed() = Unit + } + ) + } + + override fun onCreateView( + inflater: LayoutInflater, + container: ViewGroup?, + savedInstanceState: Bundle? + ): View = inflater.inflate(R.layout.fragment_sketch_to_ui, container, false) + + override fun onViewCreated(view: View, savedInstanceState: Bundle?) { + bindViews(view) + applyStatusBarColor() + setupToolbar() + setupClickListeners() + resetTransientState() + refreshTarget() + } + + fun refreshTarget() { + val current = editorService()?.getCurrentFile() + if (SketchToUiState.layoutFilePath == null && current != null && current.isLikelyLayoutXml()) { + SketchToUiState.setLayoutFile(current.absolutePath, current.name) + } + } + + private fun bindViews(view: View) { + toolbar = view.findViewById(R.id.toolbar) + imageView = view.findViewById(R.id.imageView) + guidelinesView = view.findViewById(R.id.guidelinesView) + detectButton = view.findViewById(R.id.detectButton) + updateButton = view.findViewById(R.id.updateButton) + saveButton = view.findViewById(R.id.saveButton) + imageView.onMatrixChangeListener = { matrix -> + guidelinesView.updateMatrix(matrix) + } + guidelinesView.onGuidelinesChanged = { leftPct, rightPct -> + leftGuidePct = leftPct + rightGuidePct = rightPct + } + } + + private fun setupToolbar() { + toolbar.title = getString(R.string.title_generate_xml) + toolbar.setNavigationOnClickListener { + requireActivity().onBackPressedDispatcher.onBackPressed() + } + } + + @Suppress("DEPRECATION") + private fun applyStatusBarColor() { + val color = MaterialColors.getColor(toolbar, com.google.android.material.R.attr.colorPrimary) + requireActivity().window.statusBarColor = color + } + + private fun setupClickListeners() { + imageView.setOnClickListener { openImagePicker() } + imageView.onImageTapListener = ::handleImageTap + detectButton.setOnClickListener { + runDetection() + } + updateButton.setOnClickListener { + confirmUpdate() + } + saveButton.setOnClickListener { + saveGeneratedXml() + } + } + + private fun openImagePicker() { + val intent = Intent(Intent.ACTION_OPEN_DOCUMENT).apply { + addCategory(Intent.CATEGORY_OPENABLE) + type = "image/*" + } + pickImageLauncher.launch(intent) + } + + private fun loadImage(uri: Uri) { + viewLifecycleOwner.lifecycleScope.launch { + resetTransientState() + setBusy(true) + try { + val prepared = PrepareImageUC(requireContext().contentResolver)(uri).getOrElse { error -> + toast(getString(R.string.error_detection_failed, error.message ?: error.javaClass.simpleName), long = true) + return@launch + } + selectedBitmap = prepared.bitmap + leftGuidePct = prepared.leftPct + rightGuidePct = prepared.rightPct + generatedXml = null + detectionVisualizer.clearCache() + renderImage() + guidelinesView.setImageDimensions(prepared.bitmap.width, prepared.bitmap.height) + guidelinesView.updateGuidelines(leftGuidePct, rightGuidePct) + } catch (error: Throwable) { + Log.w(TAG, "Failed to load image", error) + toast(getString(R.string.error_detection_failed, error.message ?: error.javaClass.simpleName), long = true) + } finally { + setBusy(false) + updateButtonState() + } + } + } + + private fun runDetection() { + val bitmap = selectedBitmap + if (bitmap == null) { + toast(getString(R.string.error_no_image)) + return + } + + viewLifecycleOwner.lifecycleScope.launch { + setBusy(true) + try { + val result = withContext(Dispatchers.Default) { + runComputerVisionPipeline(bitmap) + } + + result + .onSuccess { output -> + generatedXml = output.generatedXml + currentDetections = output.detections + parsedAnnotations = output.annotations + pendingImagePlaceholderId = null + selectedImagesByPlaceholderId = emptyMap() + renderDetections(output.detections) + } + .onFailure { error -> + toast(getString(R.string.error_detection_failed, error.message ?: error.javaClass.simpleName), long = true) + } + } catch (error: Throwable) { + Log.w(TAG, "Detection failed", error) + toast(getString(R.string.error_detection_failed, error.message ?: error.javaClass.simpleName), long = true) + } finally { + setBusy(false) + updateButtonState() + } + } + } + + private suspend fun runComputerVisionPipeline(bitmap: Bitmap): Result { + val hostContext = requireContext() + val pluginContext = SketchToUiPlugin.getPluginAndroidContext() ?: hostContext + val ocrSource = OcrSource(pluginContext) + val repository = VisionRepositoryImpl( + assetManager = hostContext.assets, + yoloModelSource = YoloModelSource(), + ocrSource = ocrSource + ) + + return try { + repository.initModel().getOrThrow() + val visionResult = RunVisionUC( + repository = repository, + boxResolver = GenericBoxResolver(), + regionOcrProcessor = RegionOcrProcessor(ocrSource) + )( + bitmap = bitmap, + leftPct = leftGuidePct, + rightPct = rightGuidePct + ) { operation -> + view?.post { announceStatus(operation.toStatusText()) } + }.getOrThrow() + + val (layoutXml, stringsXml) = GenerateXmlUC()( + detections = visionResult.detections, + annotations = visionResult.annotations, + selectedImagesByPlaceholderId = selectedImagesByPlaceholderId.mapValues { it.value.drawableReference }, + sourceImageWidth = bitmap.width, + sourceImageHeight = bitmap.height, + targetDpWidth = TARGET_DP_WIDTH, + targetDpHeight = TARGET_DP_HEIGHT + ).getOrThrow() + + Result.success( + PipelineOutput( + detectionCount = visionResult.detections.size, + detections = visionResult.detections, + annotations = visionResult.annotations, + generatedXml = GeneratedXml(layoutXml, stringsXml) + ) + ) + } catch (error: Throwable) { + Result.failure(error) + } finally { + repository.release() + } + } + + private fun CvOperation.toStatusText(): String = when (this) { + CvOperation.InitializingModel -> getString(R.string.status_initializing_model) + CvOperation.RunningYolo -> getString(R.string.status_running_yolo) + CvOperation.RunningOcr -> getString(R.string.status_running_ocr) + CvOperation.MergingDetections -> getString(R.string.status_merging_detections) + CvOperation.GeneratingXml -> getString(R.string.status_generating_xml) + CvOperation.SavingFile -> getString(R.string.status_saving_file) + CvOperation.Idle -> getString(R.string.status_detecting) + } + + private fun confirmUpdate() { + val xml = generatedXml + if (xml == null) { + toast(getString(R.string.error_no_generated_xml)) + return + } + + val targetName = SketchToUiState.layoutFileName + ?: editorService()?.getCurrentFile()?.name + ?: "current layout" + + AlertDialog.Builder(requireContext()) + .setTitle(R.string.confirm_update_title) + .setMessage(getString(R.string.confirm_update_message, targetName)) + .setNegativeButton(R.string.no, null) + .setPositiveButton(R.string.yes) { dialog, _ -> + dialog.dismiss() + applyGeneratedResult(xml) + } + .show() + } + + private fun applyGeneratedResult(xml: GeneratedXml) { + val target = SketchToUiState.layoutFilePath?.let(::File) + ?: editorService()?.getCurrentFile() + + if (target == null) { + toast(getString(R.string.error_no_layout_file), long = true) + return + } + + val stringsWritten = GeneratedStringsWriter(projectService(), fileService()).write(xml.stringsXml) + val layoutWritten = replaceOpenEditorContent(target, xml.layoutXml) || + fileService()?.writeFile(target, xml.layoutXml) == true + + if (layoutWritten && stringsWritten) { + resetTransientState() + toast(getString(R.string.status_updated)) + requireActivity().onBackPressedDispatcher.onBackPressed() + } else { + toast(getString(R.string.error_update_failed), long = true) + } + } + + private fun replaceOpenEditorContent(target: File, layoutXml: String): Boolean { + val service = editorService() ?: return false + val current = service.getCurrentFile() ?: return false + if (current.canonicalPath != target.canonicalPath) return false + + val lineCount = service.getLineCount(current) + if (lineCount <= 0) return false + + val lastLine = lineCount - 1 + val lastColumn = service.getLineText(current, lastLine)?.length ?: 0 + return service.replaceRange( + current, + SelectionRange(0, 0, lastLine, lastColumn), + layoutXml + ) + } + + private fun saveGeneratedXml() { + val xml = generatedXml + if (xml == null) { + toast(getString(R.string.error_no_generated_xml)) + } else { + XmlFileManager(requireContext()) + .saveXmlToDownloads(xml.layoutXml) + .onSuccess { fileName -> + toast(getString(R.string.msg_saved_to_downloads, fileName), long = true) + } + .onFailure { error -> + toast(getString(R.string.msg_error_saving_file, error.message), long = true) + } + } + } + + private fun handleImageTap(imageX: Float, imageY: Float): Boolean { + val tappedDeleteId = detectionVisualizer.getTappedDeleteIconId(imageX, imageY) + if (tappedDeleteId != null) { + removePlaceholderImage(tappedDeleteId) + return true + } + + val placeholder = findImagePlaceholderAt(imageX, imageY) ?: return false + pendingImagePlaceholderId = resolvePlaceholderId(placeholder) + pickPlaceholderImageLauncher.launch("image/*") + return true + } + + private fun loadPlaceholderImage(uri: Uri) { + val placeholderId = pendingImagePlaceholderId ?: return + val bitmap = selectedBitmap ?: return + + viewLifecycleOwner.lifecycleScope.launch { + setBusy(true) + try { + importPlaceholderImageUC(uri, SketchToUiState.layoutFilePath, placeholderId) + .onSuccess { importedDrawable -> + selectedImagesByPlaceholderId = selectedImagesByPlaceholderId + + (placeholderId to SelectedImportedImage(importedDrawable.resourceName, importedDrawable.drawableReference)) + pendingImagePlaceholderId = null + regenerateGeneratedXml(bitmap) + renderDetections(currentDetections) + toast(getString(R.string.msg_placeholder_image_selected)) + } + .onFailure { error -> + pendingImagePlaceholderId = null + toast("Image import failed: ${error.message}", long = true) + } + } finally { + setBusy(false) + updateButtonState() + } + } + } + + private fun removePlaceholderImage(placeholderId: String) { + val importedImageInfo = selectedImagesByPlaceholderId[placeholderId] ?: return + val bitmap = selectedBitmap ?: return + + viewLifecycleOwner.lifecycleScope.launch { + setBusy(true) + try { + removePlaceholderImageUC(SketchToUiState.layoutFilePath, importedImageInfo.resourceName) + .onSuccess { + selectedImagesByPlaceholderId = selectedImagesByPlaceholderId - placeholderId + pendingImagePlaceholderId = null + regenerateGeneratedXml(bitmap) + renderDetections(currentDetections) + toast(getString(R.string.msg_image_removed)) + } + .onFailure { error -> + toast("Failed to clean up image file: ${error.message}", long = true) + } + } finally { + setBusy(false) + updateButtonState() + } + } + } + + private fun regenerateGeneratedXml(bitmap: Bitmap) { + GenerateXmlUC()( + detections = currentDetections, + annotations = parsedAnnotations, + selectedImagesByPlaceholderId = selectedImagesByPlaceholderId.mapValues { it.value.drawableReference }, + sourceImageWidth = bitmap.width, + sourceImageHeight = bitmap.height, + targetDpWidth = TARGET_DP_WIDTH, + targetDpHeight = TARGET_DP_HEIGHT + ).onSuccess { (layoutXml, stringsXml) -> + generatedXml = GeneratedXml(layoutXml, stringsXml) + }.onFailure { error -> + toast("XML generation failed: ${error.message}", long = true) + } + } + + private fun findImagePlaceholderAt(imageX: Float, imageY: Float): DetectionResult? { + return currentDetections + .getSortedPlaceholders() + .firstOrNull { it.boundingBox.contains(imageX, imageY) } + } + + private fun resolvePlaceholderId(detection: DetectionResult): String { + val index = currentDetections.getSortedPlaceholders().indexOf(detection) + return "ph_${index.coerceAtLeast(0)}" + } + + private fun updateButtonState() { + detectButton.isEnabled = selectedBitmap != null + updateButton.isEnabled = generatedXml != null + saveButton.isEnabled = generatedXml != null + } + + private fun resetTransientState() { + selectedBitmap = null + generatedXml = null + currentDetections = emptyList() + parsedAnnotations = emptyMap() + pendingImagePlaceholderId = null + selectedImagesByPlaceholderId = emptyMap() + leftGuidePct = DEFAULT_LEFT_GUIDE_PCT + rightGuidePct = DEFAULT_RIGHT_GUIDE_PCT + detectionVisualizer.clearCache() + imageView.setImageDrawable(null) + imageView.isEnabled = true + guidelinesView.setImageDimensions(0, 0) + guidelinesView.updateGuidelines(leftGuidePct, rightGuidePct) + updateButtonState() + } + + private fun renderImage() { + imageView.setImageBitmap(selectedBitmap) + } + + private fun renderDetections(detections: List) { + val bitmap = selectedBitmap ?: return + val visualizedBitmap = detectionVisualizer.visualize( + bitmap = bitmap, + detections = detections, + selectedPlaceholderIds = selectedImagesByPlaceholderId.keys + ) + imageView.setImageBitmap(visualizedBitmap) + guidelinesView.setImageDimensions(bitmap.width, bitmap.height) + guidelinesView.updateGuidelines(leftGuidePct, rightGuidePct) + } + + private fun setBusy(isBusy: Boolean) { + imageView.isEnabled = !isBusy + detectButton.isEnabled = !isBusy && selectedBitmap != null + updateButton.isEnabled = !isBusy && generatedXml != null + saveButton.isEnabled = !isBusy && generatedXml != null + } + + private fun announceStatus(message: String) { + view?.announceForAccessibility(message) + } + + private fun editorService(): IdeEditorService? = + PluginFragmentHelper.getServiceRegistry(SketchToUiPlugin.PLUGIN_ID) + ?.get(IdeEditorService::class.java) + + private fun fileService(): IdeFileService? = + PluginFragmentHelper.getServiceRegistry(SketchToUiPlugin.PLUGIN_ID) + ?.get(IdeFileService::class.java) + + private fun projectService(): IdeProjectService? = + PluginFragmentHelper.getServiceRegistry(SketchToUiPlugin.PLUGIN_ID) + ?.get(IdeProjectService::class.java) + + private fun File.isLikelyLayoutXml(): Boolean = + extension.equals("xml", ignoreCase = true) && + invariantSeparatorsPath.contains("/res/layout") + + private fun toast(message: String, long: Boolean = false) { + Toast.makeText(requireContext(), message, if (long) Toast.LENGTH_LONG else Toast.LENGTH_SHORT).show() + } + + private data class PipelineOutput( + val detectionCount: Int, + val detections: List, + val annotations: Map, + val generatedXml: GeneratedXml + ) + + private data class GeneratedXml( + val layoutXml: String, + val stringsXml: String + ) + + companion object { + private const val TAG = "SketchToUiFragment" + private const val TARGET_DP_WIDTH = 360 + private const val TARGET_DP_HEIGHT = 640 + private const val DEFAULT_LEFT_GUIDE_PCT = 0.2f + private const val DEFAULT_RIGHT_GUIDE_PCT = 0.8f + } +} diff --git a/sketch-to-ui-plugin/src/main/res/drawable/baseline_feedback_64.xml b/sketch-to-ui-plugin/src/main/res/drawable/baseline_feedback_64.xml new file mode 100644 index 0000000..b2466db --- /dev/null +++ b/sketch-to-ui-plugin/src/main/res/drawable/baseline_feedback_64.xml @@ -0,0 +1,10 @@ + + + diff --git a/sketch-to-ui-plugin/src/main/res/drawable/ic_arrow_back.xml b/sketch-to-ui-plugin/src/main/res/drawable/ic_arrow_back.xml new file mode 100644 index 0000000..5ff8ddb --- /dev/null +++ b/sketch-to-ui-plugin/src/main/res/drawable/ic_arrow_back.xml @@ -0,0 +1,10 @@ + + + + diff --git a/sketch-to-ui-plugin/src/main/res/drawable/ic_arrow_left.xml b/sketch-to-ui-plugin/src/main/res/drawable/ic_arrow_left.xml new file mode 100644 index 0000000..615b651 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/res/drawable/ic_arrow_left.xml @@ -0,0 +1,14 @@ + + + + + \ No newline at end of file diff --git a/sketch-to-ui-plugin/src/main/res/drawable/ic_computer_vision.xml b/sketch-to-ui-plugin/src/main/res/drawable/ic_computer_vision.xml new file mode 100644 index 0000000..0bac685 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/res/drawable/ic_computer_vision.xml @@ -0,0 +1,22 @@ + + + + + + \ No newline at end of file diff --git a/sketch-to-ui-plugin/src/main/res/drawable/ic_placeholder_delete.xml b/sketch-to-ui-plugin/src/main/res/drawable/ic_placeholder_delete.xml new file mode 100644 index 0000000..20cebf7 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/res/drawable/ic_placeholder_delete.xml @@ -0,0 +1,10 @@ + + + diff --git a/sketch-to-ui-plugin/src/main/res/drawable/ic_placeholder_upload.xml b/sketch-to-ui-plugin/src/main/res/drawable/ic_placeholder_upload.xml new file mode 100644 index 0000000..203884b --- /dev/null +++ b/sketch-to-ui-plugin/src/main/res/drawable/ic_placeholder_upload.xml @@ -0,0 +1,10 @@ + + + diff --git a/sketch-to-ui-plugin/src/main/res/layout/fragment_sketch_to_ui.xml b/sketch-to-ui-plugin/src/main/res/layout/fragment_sketch_to_ui.xml new file mode 100644 index 0000000..7a83944 --- /dev/null +++ b/sketch-to-ui-plugin/src/main/res/layout/fragment_sketch_to_ui.xml @@ -0,0 +1,71 @@ + + + + + + + + + + + + + + +