Revamp search indexing and UI for docs

steida · steida · commit f060169dfd54 · 2025-12-19T15:38:42.000+01:00
Replaces old search algorithm to improve result ranking and relevance. Updates the search index build process, adds a new searchIndex.js template, and refines section extraction in MDX. The Search UI is simplified by removing navigation hierarchy and improving result display. Also updates navigation links and adds scroll offset for anchor targets in CSS.
diff --git a/apps/web/scripts/fix-api-reference.mts b/apps/web/scripts/fix-api-reference.mts
@@ -25,7 +25,11 @@ function rearrangeMdxFilesRecursively(dir: string) {
           `${baseName} - API reference`,
         );
       } else {
-        fixLinksInMdxFile(fullPath, "API reference");
+        const title =
+          dir === reference
+            ? "API reference"
+            : `${path.basename(dir)} - API reference`;
+        fixLinksInMdxFile(fullPath, title);
       }
     }
   }
@@ -34,30 +38,71 @@ function rearrangeMdxFilesRecursively(dir: string) {
 function fixLinksInMdxFile(filePath: string, title: string) {
   const content = fs.readFileSync(filePath, "utf-8");
   // first let's replace /page.mdx with /
-  let newContent = content.replace(/\/page.mdx/g, "");
-  newContent = newContent.replace(/\(([^)]+)\.mdx\)/g, "($1)");
+  let newContent = content.replace(/\/page\.mdx/g, "");
+  // Remove .mdx from Markdown link destinations, preserving query/hash.
+  // Examples:
+  // - [X](/docs/Foo.mdx) -> [X](/docs/Foo)
+  // - [X](/docs/Foo.mdx#bar) -> [X](/docs/Foo#bar)
+  // - [X](../Foo.mdx?x=1#bar) -> [X](../Foo?x=1#bar)
+  newContent = newContent.replace(/\]\(([^)]*?)\.mdx(?=[)#?])/g, "]($1");
 
-  // fix API reference breadcrumb link
+  // fix API reference breadcrumb link and separator
+  // Breadcrumb is the first line starting with `[API` - replace link text and separators
   newContent = newContent.replace(
-    /\[API Reference\]\([^)]*\)/g,
-    "[API reference](/docs/api-reference)",
+    /^(\[API Reference\]\([^)]*\))(.*)/m,
+    (_match, _apiLink, rest: string) => {
+      const fixedRest = rest.replace(/ \/ /g, " › ");
+      return `[API reference](/docs/api-reference)${fixedRest}`;
+    },
   );
 
-  // Remove call signatures
+  // Extract ## headings to generate sections for "On this page" navigation
+  const sections = extractSections(newContent);
+  const sectionsExport =
+    sections.length > 0
+      ? `export const sections = ${JSON.stringify(sections)};`
+      : "export const sections = [];";
+
+  // add meta tags (idempotent)
   newContent = newContent.replace(
-    /##\s*Call Signature\r?\n\s*```ts[\s\S]*?```/g,
+    /^export const metadata = \{ title: [^}]*\};\s*\r?\n\s*/,
+    "",
+  );
+  newContent = newContent.replace(
+    /^export const sections = .*;\s*\r?\n\s*/m,
     "",
   );
-
-  // add meta tags
   newContent = `export const metadata = { title: '${title}' };
-export const sections = [];
+${sectionsExport}
 	
 ${newContent}`;
 
   fs.writeFileSync(filePath, newContent);
 }
 
+/** Extract ## headings from MDX content to generate sections */
+function extractSections(
+  content: string,
+): Array<{ id: string; title: string }> {
+  const sections: Array<{ id: string; title: string }> = [];
+  // Match ## headings (not ### or deeper)
+  const headingRegex = /^## (.+)$/gm;
+  let match;
+  while ((match = headingRegex.exec(content)) !== null) {
+    const title = match[1].trim();
+    // Generate id from title (kebab-case)
+    const id = title
+      .toLowerCase()
+      .replace(/[^a-z0-9\s-]/g, "")
+      .replace(/\s+/g, "-")
+      .replace(/-+/g, "-");
+    if (id) {
+      sections.push({ id, title });
+    }
+  }
+  return sections;
+}
+
 // Run the script
 rearrangeMdxFilesRecursively(reference);
 
diff --git a/apps/web/src/components/Search.tsx b/apps/web/src/components/Search.tsx
@@ -11,7 +11,6 @@ import clsx from "clsx";
 import { usePathname, useRouter, useSearchParams } from "next/navigation";
 import {
   forwardRef,
-  Fragment,
   Suspense,
   useCallback,
   useEffect,
@@ -21,7 +20,6 @@ import {
 } from "react";
 import Highlighter from "react-highlight-words";
 
-import { navigation } from "@/lib/navigation";
 import { type Result } from "@/mdx/search.mjs";
 
 type EmptyObject = Record<string, never>;
@@ -83,7 +81,7 @@ function useAutocomplete({ close }: { close: () => void }) {
             {
               sourceId: "documentation",
               getItems() {
-                return search(query, { limit: 5 });
+                return search(query);
               },
               getItemUrl({ item }) {
                 return item.url;
@@ -179,20 +177,13 @@ function SearchResult({
 }) {
   const id = useId();
 
-  const sectionTitle = navigation.find((section) =>
-    section.links.find((link) => link.href === result.url.split("#")[0]),
-  )?.title;
-  const hierarchy = [sectionTitle, result.pageTitle].filter(
-    (x): x is string => typeof x === "string",
-  );
-
   return (
     <li
       className={clsx(
         "group block cursor-default px-4 py-3 aria-selected:bg-zinc-50 dark:aria-selected:bg-zinc-800/50",
         resultIndex > 0 && "border-t border-zinc-100 dark:border-zinc-800",
       )}
-      aria-labelledby={`${id}-hierarchy ${id}-title`}
+      aria-labelledby={`${id}-title`}
       {...autocomplete.getItemProps({
         item: result,
         source: collection.source,
@@ -203,35 +194,8 @@ function SearchResult({
         aria-hidden="true"
         className="text-sm font-medium text-zinc-900 group-aria-selected:text-blue-500 dark:text-white"
       >
-        {result.url.startsWith("/docs/api-reference") ? (
-          <span className="mr-1.5">API Reference /</span>
-        ) : (
-          <></>
-        )}
         <HighlightQuery text={result.title} query={query} />
       </div>
-      {hierarchy.length > 0 && (
-        <div
-          id={`${id}-hierarchy`}
-          aria-hidden="true"
-          className="text-2xs mt-1 truncate whitespace-nowrap text-zinc-500"
-        >
-          {hierarchy.map((item, itemIndex, items) => (
-            <Fragment key={itemIndex}>
-              <HighlightQuery text={item} query={query} />
-              <span
-                className={
-                  itemIndex === items.length - 1
-                    ? "sr-only"
-                    : "mx-2 text-zinc-300 dark:text-zinc-700"
-                }
-              >
-                /
-              </span>
-            </Fragment>
-          ))}
-        </div>
-      )}
       <div className="text-2xs mt-1 truncate whitespace-nowrap text-zinc-500">
         <HighlightQuery text={result.url} query={query} />
       </div>
@@ -408,7 +372,7 @@ function SearchDialog({
               />
               <div
                 ref={panelRef}
-                className="border-t border-zinc-200 bg-white empty:hidden dark:border-zinc-100/5 dark:bg-white/2.5"
+                className="max-h-[60vh] overflow-y-auto border-t border-zinc-200 bg-white empty:hidden dark:border-zinc-100/5 dark:bg-white/2.5"
                 {...autocomplete.getPanelProps({})}
               >
                 {autocompleteState.isOpen && (
diff --git a/apps/web/src/lib/navigation.ts b/apps/web/src/lib/navigation.ts
@@ -21,7 +21,7 @@ export const navigation: Array<NavGroup> = [
       },
       {
         title: "Task",
-        href: "/docs/api-reference/common/Task/interfaces/Task",
+        href: "/docs/api-reference/common/Task/type-aliases/Task",
       },
       {
         title: "Type",
diff --git a/apps/web/src/mdx/search.mjs b/apps/web/src/mdx/search.mjs
@@ -11,6 +11,10 @@ import { SKIP, visit } from "unist-util-visit";
 import * as url from "url";
 
 const __filename = url.fileURLToPath(import.meta.url);
+const searchIndexPath = path.resolve(
+  path.dirname(__filename),
+  "./searchIndex.js",
+);
 const processor = remark().use(remarkMdx).use(extractSections);
 const slugify = slugifyWithCounter();
 
@@ -30,14 +34,20 @@ function extractSections() {
     slugify.reset();
 
     visit(tree, (node) => {
-      if (node.type === "heading" || node.type === "paragraph") {
+      if (node.type === "heading" && node.depth <= 2) {
         let content = toString(excludeObjectExpressions(node));
-        if (node.type === "heading" && node.depth <= 2) {
-          let hash = node.depth === 1 ? null : slugify(content);
-          sections.push([content, hash, []]);
-        } else {
-          sections.at(-1)?.[2].push(content);
-        }
+        let hash = node.depth === 1 ? null : slugify(content);
+        sections.push([content, hash, []]);
+        return SKIP;
+      }
+      // Extract text from paragraphs, table cells, list items, etc.
+      if (
+        node.type === "paragraph" ||
+        node.type === "tableCell" ||
+        node.type === "listItem"
+      ) {
+        let content = toString(excludeObjectExpressions(node));
+        sections.at(-1)?.[2].push(content);
         return SKIP;
       }
     });
@@ -76,53 +86,12 @@ export default function Search(nextConfig = {}) {
               return { url, sections };
             });
 
-            // When this file is imported within the application
-            // the following module is loaded:
-            return `
-              import FlexSearch from 'flexsearch'
-
-              let sectionIndex = new FlexSearch.Document({
-                tokenize: 'full',
-                document: {
-                  id: 'url',
-                  index: 'content',
-                  store: ['title', 'pageTitle'],
-                },
-                context: {
-                  resolution: 9,
-                  depth: 2,
-                  bidirectional: true
-                }
-              })
-
-              let data = ${JSON.stringify(data)}
-
-              for (let { url, sections } of data) {
-                for (let [title, hash, content] of sections) {
-                  sectionIndex.add({
-                    url: url + (hash ? ('#' + hash) : ''),
-                    title,
-                    content: [title, ...content].join('\\n'),
-                    pageTitle: hash ? sections[0][0] : undefined,
-                  })
-                }
-              }
-
-              export function search(query, options = {}) {
-                let result = sectionIndex.search(query, {
-                  ...options,
-                  enrich: true,
-                })
-                if (result.length === 0) {
-                  return []
-                }
-                return result[0].result.map((item) => ({
-                  url: item.id,
-                  title: item.doc.title,
-                  pageTitle: item.doc.pageTitle,
-                }))
-              }
-            `;
+            // Read the search index template and inject the data
+            const template = fs.readFileSync(searchIndexPath, "utf8");
+            return template.replace(
+              'const data = "DATA_PLACEHOLDER";',
+              `const data = ${JSON.stringify(data)};`,
+            );
           }),
         ],
       });
diff --git a/apps/web/src/mdx/searchIndex.js b/apps/web/src/mdx/searchIndex.js
diff --git a/apps/web/src/styles/tailwind.css b/apps/web/src/styles/tailwind.css

Original file line number	Diff line number	Diff line change
`@@ -21,7 +21,7 @@ export const navigation: Array<NavGroup> = [`
`21`	`21`	`},`
`22`	`22`	`{`
`23`	`23`	`title: "Task",`
`24`		`- href: "/docs/api-reference/common/Task/interfaces/Task",`
	`24`	`+ href: "/docs/api-reference/common/Task/type-aliases/Task",`
`25`	`25`	`},`
`26`	`26`	`{`
`27`	`27`	`title: "Type",`