Add visual highlighting for web math navigation

projectDocs/dev/developerGuide/developerGuide.md

@@ -1625,6 +1625,7 @@ Please see the `EventExtensionPoints` class documentation for more information,
 | `Action` | `post_foregroundChange` | the foreground NVDAObject has changed. |
 | `Action` | `post_caretMove` | a physical caret has moved. |
 | `Action` | `post_browseModeMove` | a virtual caret has moved. |
+| `Action` | `post_mathNavigation` | the math navigation position has changed. |
 | `Action` | `post_reviewMove` | the position of the review cursor has changed. |
 | `Action` | `post_mouseMove` | the mouse has moved. |
 | `Action` | `post_coreCycle` | the end of each core cycle has been reached. |

source/NVDAObjects/IAccessible/ia2Web.py

@@ -8,6 +8,7 @@
 from typing import (
 	Generator,
 	Optional,
+	TYPE_CHECKING,
 	Tuple,
 )
 import re
@@ -35,6 +36,10 @@
 import config
 import NVDAObjects
 
+if TYPE_CHECKING:
+	from locationHelper import RectLTRB
+	from mathPres._mathMlNode import MathMlNodePath, MathMlNodeRectInfo
+
 
 class IA2WebAnnotationTarget(AnnotationTarget):
 	def __init__(self, target: IAccessible):
@@ -331,6 +336,56 @@ class EditorChunk(Ia2Web):
 
 
 class Math(Ia2Web):
+	def _getMathElementChildren(self, obj: NVDAObjects.NVDAObject) -> tuple[IAccessible, ...]:
+		return tuple(
+			child
+			for child in obj.children
+			if isinstance(child, IAccessible) and child.IA2Attributes.get("tag")
+		)
+
+	def _getMathNodeMapRoot(self) -> IAccessible:
+		if self.IA2Attributes.get("tag") == "math":
+			return self
+		mathChildren = tuple(
+			child for child in self._getMathElementChildren(self) if child.IA2Attributes.get("tag") == "math"
+		)
+		return mathChildren[0] if len(mathChildren) == 1 else self
+
+	def _getMathNodeRectFromObj(self, obj: IAccessible) -> "RectLTRB | None":
+		if obj.hasIrrelevantLocation:
+			return None
+		location = obj.location
+		if not location or not location.width or not location.height:
+			return None
+		return location.toLTRB()
+
+	def _getMathNodeInfoByPath(self) -> dict["MathMlNodePath", "MathMlNodeRectInfo"]:
+		"""Map MathML element paths to tag names and screen rectangles for this IA2 math subtree.
+
+		Paths are tuples where each entry indicates an index of a child node to be traversed from the root.
+		"""
+		# Avoid importing mathPres at startup.
+		from mathPres._mathMlNode import MathMlNodeRectInfo
+
+		nodeInfoByPath: dict["MathMlNodePath", "MathMlNodeRectInfo"] = {}
+		stack: list[tuple[IAccessible, "MathMlNodePath"]] = [
+			(self._getMathNodeMapRoot(), ()),
+		]
+		visitedCount = 0
+		while stack:
+			obj, path = stack.pop()
+			visitedCount += 1
+			tag = obj.IA2Attributes.get("tag")
+			if tag and (rect := self._getMathNodeRectFromObj(obj)):
+				nodeInfoByPath[path] = MathMlNodeRectInfo(path=path, tag=tag, rect=rect)
+			children = self._getMathElementChildren(obj)
+			stack.extend((child, path + (index,)) for index, child in enumerate(children))
+		log.debug(
+			f"Math highlight built IA2 path map with {len(nodeInfoByPath)} usable rectangles "
+			f"after visiting {visitedCount} MathML element objects",
+		)
+		return nodeInfoByPath
+
 	def _get_mathMl(self):
 		# Chromium browsers now expose a 'math' IAccessible2 attribute,
 		# which contains all the raw MathML.

source/browseMode.py

@@ -712,7 +712,7 @@ def _activatePosition(self, obj=None):
 			import mathPres
 
 			try:
-				return mathPres.interactWithMathMl(obj.mathMl)
+				return mathPres.interactWithMathMl(obj.mathMl, sourceObj=obj)
 			except (NotImplementedError, LookupError):
 				pass
 			return

source/globalCommands.py

@@ -4818,20 +4818,28 @@ def script_toggleConfigProfileTriggers(self, gesture):
 	def script_interactWithMath(self, gesture):
 		import mathPres
 
-		mathMl = mathPres.getMathMlFromTextInfo(api.getReviewPosition())
+		reviewPosition = api.getReviewPosition()
+		mathMl = mathPres.getMathMlFromTextInfo(reviewPosition)
+		sourceObj = None
 		if not mathMl:
 			obj = api.getNavigatorObject()
 			if obj.role == controlTypes.Role.MATH:
 				try:
 					mathMl = obj.mathMl
+					sourceObj = obj
 				except (NotImplementedError, LookupError):
 					mathMl = None
+		else:
+			try:
+				sourceObj = reviewPosition.NVDAObjectAtStart
+			except (NotImplementedError, LookupError):
+				pass
 		if not mathMl:
 			# Translators: Reported when the user attempts math interaction
 			# with something that isn't math.
 			ui.message(_("Not math"))
 			return
-		mathPres.interactWithMathMl(mathMl)
+		mathPres.interactWithMathMl(mathMl, sourceObj=sourceObj)
 
 	@script(
 		# Translators: Describes a command.

source/mathPres/MathCAT/MathCAT.py

@@ -12,7 +12,7 @@
 	windll,
 )
 from os import path
-from typing import Type
+from typing import TYPE_CHECKING, Type
 
 import braille
 import braille.regions.base
@@ -22,6 +22,7 @@
 import libmathcat_py as libmathcat
 import speech
 import ui
+import vision
 import winKernel
 import winUser
 from api import getClipData
@@ -46,9 +47,18 @@
 import mathPres
 from .localization import getLanguageToUse
 from .navCommands import NAV_COMMANDS
+from ._navNodeMapping import (
+	_NAV_NODE_ID_PREFIX,
+	prepareMathMlForNavigation,
+	removeSyntheticIdsFromMathMl,
+)
 from .preferences import applyUserPreferences
 from .speech import convertSSMLTextForNVDA
 
+if TYPE_CHECKING:
+	from locationHelper import RectLTRB
+	from NVDAObjects import NVDAObject
+
 
 class MathCATError(Exception):
 	"""MathCAT failure, including Rust panics from PyO3."""
@@ -83,28 +93,80 @@ def __init__(
 		self,
 		provider: mathPres.MathPresentationProvider | None = None,
 		mathMl: str | None = None,
-	):
+		sourceObj: "NVDAObject | None" = None,
+	) -> None:
 		"""Initialize the MathCATInteraction object.
 
 		:param provider: Optional presentation provider.
 		:param mathMl: Optional initial MathML string.
+		:param sourceObj: Optional source object containing the math.
 		"""
-		super(MathCATInteraction, self).__init__(provider=provider, mathMl=mathMl)
+		super().__init__(provider=provider, mathMl=mathMl, sourceObj=sourceObj)
+		self._mathMlForNavigation, self._mathNodeRectsById = prepareMathMlForNavigation(
+			mathMl or "<math></math>",
+			sourceObj,
+		)
+		self._shouldUpdateMathHighlight: bool = False
 		if mathMl is None:
 			self.initMathML = "<math></math>"
 		else:
 			self.initMathML = mathMl
 
 	def reportFocus(self) -> None:
 		"""Calls MathCAT's ZoomIn command and speaks the resulting text."""
+		self._shouldUpdateMathHighlight = False
 		super(MathCATInteraction, self).reportFocus()
 		try:
 			text: str = libmathcat.DoNavigateCommand("ZoomIn")
 			speech.speak(convertSSMLTextForNVDA(text))
+			self._shouldUpdateMathHighlight = True
 		except Exception:
 			log.exception()
 			# Translators: this message reports an error in starting navigation of math.
 			ui.message(pgettext("math", "Error in starting navigation of math."))
+			self._clearMathHighlight()
+
+	def event_gainFocus(self) -> None:
+		"""Update the math highlight after inherited focus handling has completed."""
+		super().event_gainFocus()
+		if self._shouldUpdateMathHighlight:
+			self._shouldUpdateMathHighlight = False
+			self._updateMathHighlight()
+
+	def _getHighlightRect(self) -> "RectLTRB | None":
+		"""Get the navigation rectangle for a supported web math source object."""
+		sourceObj = self.sourceObj
+		if not sourceObj:
+			return None
+		# Avoid importing ia2Web at startup.
+		from NVDAObjects.IAccessible.ia2Web import Math as Ia2WebMath
+
+		if not isinstance(sourceObj, Ia2WebMath):
+			return None
+		try:
+			nodeId = libmathcat.GetNavigationMathMLId()[0]
+		except Exception:
+			log.debugWarning("Error getting MathCAT navigation node id", exc_info=True)
+			return None
+		if nodeId in self._mathNodeRectsById:
+			return self._mathNodeRectsById[nodeId]
+		if nodeId.startswith(_NAV_NODE_ID_PREFIX):
+			log.debug(
+				f"Math highlight found synthetic MathML id {nodeId!r}, but it has no mapped IA2 rectangle",
+			)
+		log.debug(f"Math highlight falling back to source rectangle for node id {nodeId!r}")
+		if sourceObj.hasIrrelevantLocation:
+			return None
+		location = sourceObj.location
+		return location.toLTRB() if location else None
+
+	def _updateMathHighlight(self) -> None:
+		if vision.handler:
+			vision.handler.handleMathNavigation(self._getHighlightRect())
+
+	def _clearMathHighlight(self) -> None:
+		if vision.handler:
+			vision.handler.handleMathNavigation(None)
 
 	def getBrailleRegions(
 		self,
@@ -136,6 +198,9 @@ def _doNavigateCommand(self, commandName: str) -> None:
 			log.exception()
 			# Translators: this message alerts users to an error in navigating math.
 			ui.message(pgettext("math", "Error in navigating math"))
+			self._clearMathHighlight()
+		else:
+			self._updateMathHighlight()
 
 		self._updateBraille()
 
@@ -212,7 +277,7 @@ def script_rawdataToClip(self, gesture: KeyboardInputGesture) -> None:
 					mathml = self.initMathML
 				if copyAs == "speech":
 					# save the old MathML, set the navigation MathML as MathMl, get the speech, then reset the MathML
-					savedMathML: str = self.initMathML
+					savedMathML: str = self._mathMlForNavigation
 					savedTTS: str = libmathcat.GetPreference("TTS")
 					if savedMathML == "":  # shouldn't happen
 						raise Exception("Internal error -- MathML not set for copy")
@@ -235,12 +300,11 @@ def script_rawdataToClip(self, gesture: KeyboardInputGesture) -> None:
 
 	# not a perfect match sequence, but should capture normal MathML
 	_mathTagHasNameSpace: re.Pattern = re.compile("<math .*?xmlns.+?>")
-	_hasAddedId: re.Pattern = re.compile(" id='[^'].+' data-id-added='true'")
-	_hasDataAttr: re.Pattern = re.compile(" data-[^=]+='[^']*'")
+	_hasDataAttr: re.Pattern = re.compile(r""" data-[^=]+=(['"]).*?\1""")
 
 	def _wrapMathMLForClipBoard(self, text: str) -> str:
 		"""Cleanup the MathML a little."""
-		text = re.sub(self._hasAddedId, "", text)
+		text = removeSyntheticIdsFromMathMl(text)
 		mathMLWithNS: str = re.sub(self._hasDataAttr, "", text)
 		if not re.match(self._mathTagHasNameSpace, mathMLWithNS):
 			mathMLWithNS = mathMLWithNS.replace(
@@ -413,11 +477,45 @@ def getBrailleForMathMl(self, mathml: str) -> str:
 			ui.message(pgettext("math", "Error in brailling math."))
 			return ""
 
-	def interactWithMathMl(self, mathml: str) -> None:
-		"""Interact with a MathML string, creating a MathCATInteraction object.
+	def _startMathInteraction(
+		self,
+		mathml: str,
+		sourceObj: "NVDAObject | None" = None,
+	) -> None:
+		"""Start interacting with a MathML string.
+
+		This is a helper called by ``interactWithMathMl`` and ``interactWithMathMlFromSource``.
 
 		:param mathml: The MathML representing the math to interact with.
+		:param sourceObj: Optional source object containing the math.
 		"""
-		interaction = MathCATInteraction(provider=self, mathMl=mathml)
+		interaction = MathCATInteraction(provider=self, mathMl=mathml, sourceObj=sourceObj)
+		try:
+			libmathcat.SetMathML(interaction._mathMlForNavigation)
+		except Exception:
+			log.exception("Error setting MathML for navigation")
+			# Translators: this message reports illegal MathML.
+			ui.message(pgettext("math", "Invalid MathML found."))
+			libmathcat.SetMathML("<math></math>")
+			return
 		interaction.setFocus()
 		interaction._updateBraille()
+
+	def interactWithMathMl(self, mathml: str) -> None:
+		"""Interact with a MathML string, creating a MathCATInteraction object.
+
+		:param mathml: The MathML representing the math to interact with.
+		"""
+		self._startMathInteraction(mathml, sourceObj=None)
+
+	def interactWithMathMlFromSource(
+		self,
+		mathml: str,
+		sourceObj: "NVDAObject",
+	) -> None:
+		"""Interact with MathML from the given source object.
+
+		:param mathml: The MathML representing the math to interact with.
+		:param sourceObj: The source object containing the math.
+		"""
+		self._startMathInteraction(mathml, sourceObj=sourceObj)