From 360c326b955a368f39f48a4ef86e56f3444f6f09 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 23 Oct 2026 16:11:08 -0700 Subject: [PATCH 1/5] improvement(library): single JSON-LD source, Markdown articles, and IndexNow pings --- .github/workflows/ci.yml | 36 ++++++ apps/sim/app/(landing)/blog/[slug]/page.tsx | 2 +- .../app/(landing)/changelog/[slug]/page.tsx | 2 +- .../content-post-page/content-post-page.tsx | 25 +--- .../app/(landing)/customers/[slug]/page.tsx | 2 +- apps/sim/app/library.md/[slug]/route.ts | 57 +++++++++ apps/sim/app/llms.txt/route.ts | 2 + apps/sim/lib/blog/seo.ts | 1 + apps/sim/lib/content/faq.tsx | 14 +-- apps/sim/lib/content/seo.ts | 66 +++++++--- apps/sim/lib/library/seo.ts | 18 ++- apps/sim/next.config.ts | 11 ++ .../e300fec30f4d245dc5ad832693bbc144.txt | 1 + apps/sim/scripts/test-landing-seo-e2e.ts | 69 +++++++++- scripts/ping-indexnow.test.ts | 113 +++++++++++++++++ scripts/ping-indexnow.ts | 118 ++++++++++++++++++ 16 files changed, 484 insertions(+), 53 deletions(-) create mode 100644 apps/sim/app/library.md/[slug]/route.ts create mode 100644 apps/sim/public/e300fec30f4d245dc5ad832693bbc144.txt create mode 100644 scripts/ping-indexnow.test.ts create mode 100644 scripts/ping-indexnow.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62879db98f2..b49a98311c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -636,6 +636,42 @@ jobs: fi bunx trigger.dev@4.5.16 promote "$VERSION" "${TARGET_ARGS[@]}" + # Production only, once the new app serves traffic: submit the library and comparison URLs this + # push changed to IndexNow. Best effort: nothing depends on it, and a failure never fails the run. + indexnow: + name: indexnow + needs: [promote, trigger-promote] + if: >- + !cancelled() && + github.event_name == 'push' && + github.ref == 'refs/heads/main' && + needs.promote.outputs.promoted == 'true' && + needs.trigger-promote.result == 'success' + runs-on: *runner-2vcpu + timeout-minutes: 5 + continue-on-error: true + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.4.4 + + # The diff needs only the two trees, so the previous head is fetched alone. A push that + # created the branch has an all-zero `before`, which the script skips. + - name: Submit changed URLs to IndexNow + env: + BASE_SHA: ${{ github.event.before }} + run: | + if [[ ! "$BASE_SHA" =~ ^0+$ ]]; then + git fetch --no-tags --depth=1 origin "$BASE_SHA" + fi + bun scripts/ping-indexnow.ts "$BASE_SHA" "$GITHUB_SHA" + # Build ARM64 images for GHCR (main branch only, runs in parallel with # tests). Pushes only the immutable sha tag — latest-arm64/version-arm64 # are applied by ghcr after the gate, so a failing run diff --git a/apps/sim/app/(landing)/blog/[slug]/page.tsx b/apps/sim/app/(landing)/blog/[slug]/page.tsx index 89385e4b472..c1fbaae1399 100644 --- a/apps/sim/app/(landing)/blog/[slug]/page.tsx +++ b/apps/sim/app/(landing)/blog/[slug]/page.tsx @@ -24,7 +24,7 @@ export async function generateMetadata({ const { slug } = await params const post = await getPostBySlug(slug) if (!post || post.draft) return {} - return buildPostMetadata(post) + return buildPostMetadata(post, BLOG_SECTION) } export const revalidate = 86400 diff --git a/apps/sim/app/(landing)/changelog/[slug]/page.tsx b/apps/sim/app/(landing)/changelog/[slug]/page.tsx index ec1ffb7f4bf..e8a92d79908 100644 --- a/apps/sim/app/(landing)/changelog/[slug]/page.tsx +++ b/apps/sim/app/(landing)/changelog/[slug]/page.tsx @@ -19,7 +19,7 @@ export async function generateStaticParams() { export async function generateMetadata({ params }: ChangelogEntryPageProps): Promise { const { slug } = await params const entry = await getEntryBySlug(slug) - return entry ? buildPostMetadata(entry) : {} + return entry ? buildPostMetadata(entry, CHANGELOG_SECTION) : {} } export default async function ChangelogEntryPage({ params }: ChangelogEntryPageProps) { diff --git a/apps/sim/app/(landing)/components/content-post-page/content-post-page.tsx b/apps/sim/app/(landing)/components/content-post-page/content-post-page.tsx index 9af4e9a7393..69b3b71b3c8 100644 --- a/apps/sim/app/(landing)/components/content-post-page/content-post-page.tsx +++ b/apps/sim/app/(landing)/components/content-post-page/content-post-page.tsx @@ -40,7 +40,7 @@ export function ContentPostPage({ const showUpdated = modifiedIso.slice(0, 10) !== post.date.slice(0, 10) return ( -
+
@@ -59,7 +59,6 @@ export function ContentPostPage({ sizes='(max-width: 1023px) calc(100vw - 64px), 450px' priority fetchPriority='high' - itemProp='image' unoptimized />
@@ -68,13 +67,13 @@ export function ContentPostPage({

{post.title}

{post.description}

@@ -84,11 +83,10 @@ export function ContentPostPage({ - {showUpdated ? ( + {showUpdated && ( <>
@@ -118,11 +113,8 @@ export function ContentPostPage({ href={`${basePath}/authors/${encodeURIComponent(a?.id ?? '')}`} rel='author' className='text-[var(--text-secondary)] text-xs uppercase tracking-[0.1em] hover:text-[var(--text-primary)]' - itemProp='author' - itemScope - itemType='https://schema.org/Person' > - {a?.name} + {a?.name}
))} @@ -139,7 +131,7 @@ export function ContentPostPage({
-
+
- - - - - {post.wordCount && }
) } diff --git a/apps/sim/app/(landing)/customers/[slug]/page.tsx b/apps/sim/app/(landing)/customers/[slug]/page.tsx index e8a0e9be0a7..2ce7b9f2106 100644 --- a/apps/sim/app/(landing)/customers/[slug]/page.tsx +++ b/apps/sim/app/(landing)/customers/[slug]/page.tsx @@ -21,7 +21,7 @@ export async function generateMetadata({ params }: CustomerPageProps): Promise story.slug === slug)) return {} const post = await getCustomerStoryBySlug(slug) - return post ? buildPostMetadata(post) : {} + return post ? buildPostMetadata(post, CUSTOMER_SECTION) : {} } export default async function Page({ params }: CustomerPageProps) { diff --git a/apps/sim/app/library.md/[slug]/route.ts b/apps/sim/app/library.md/[slug]/route.ts new file mode 100644 index 00000000000..0730b19e723 --- /dev/null +++ b/apps/sim/app/library.md/[slug]/route.ts @@ -0,0 +1,57 @@ +import type { NextRequest } from 'next/server' +import type { ContentMeta } from '@/lib/content/schema' +import { toSiteUrl } from '@/lib/core/utils/urls' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { getAllPostMeta, getPostSource } from '@/lib/library/registry' +import { LIBRARY_SECTION } from '@/lib/library/seo' +import { markdownResponse, section, toLlmsMarkdown } from '@/app/llms.txt/llms' + +type RouteContext = { params: Promise<{ slug: string }> } + +const NOT_FOUND_MARKDOWN = `# Article not found + +There is no library article at this address. Browse the [library](${toSiteUrl(LIBRARY_SECTION.basePath)}) or the [llms.txt](${toSiteUrl('/llms.txt')}) index. +` + +function postMarkdownBlocks(post: ContentMeta, source: string): string[] { + const authors = (post.authors.length > 0 ? post.authors : [post.author]).map((a) => a.name) + const published = post.date.slice(0, 10) + const updated = (post.updated ?? post.date).slice(0, 10) + return [ + `# ${post.title}`, + `> ${post.description}`, + [ + `URL: ${post.canonical}`, + `By ${authors.join(', ')} · Published ${published}${updated !== published ? ` · Updated ${updated}` : ''}`, + ].join('\n'), + toLlmsMarkdown(source, 0), + section( + 'FAQ', + (post.faq ?? []).map((item) => `### ${item.q}\n\n${item.a}`), + '\n\n' + ), + ] +} + +/** + * A library article as Markdown, at `/library/.md` (and at `/library/` for a client + * whose `Accept` names `text/markdown`), via the rewrites in `next.config.ts`. Drafts and unknown + * slugs are a Markdown 404. Uncached, like the HTML post page: the same body answers at the post's + * own URL, so a cache that ignores `Vary` must never keep it. + */ +export const GET = withRouteHandler(async (_request: NextRequest, context: RouteContext) => { + const { slug } = await context.params + const [posts, source] = await Promise.all([getAllPostMeta(), getPostSource(slug)]) + const post = posts.find((candidate) => candidate.slug === slug) + const response = + post && source !== null + ? markdownResponse(postMarkdownBlocks(post, source), 0) + : new Response(NOT_FOUND_MARKDOWN, { + status: 404, + headers: { 'Content-Type': 'text/markdown; charset=utf-8' }, + }) + response.headers.set('Cache-Control', 'no-store') + response.headers.set('Vary', 'Accept') + if (post) response.headers.set('Link', `<${post.canonical}>; rel="canonical"`) + return response +}) diff --git a/apps/sim/app/llms.txt/route.ts b/apps/sim/app/llms.txt/route.ts index bbd7770ccb6..e67c3ad6228 100644 --- a/apps/sim/app/llms.txt/route.ts +++ b/apps/sim/app/llms.txt/route.ts @@ -79,6 +79,8 @@ export async function GET() { ), ]), section('Library', [ + 'Each article is also served as Markdown at its URL with `.md` appended.', + '', linkLine('Library', '/library', 'Comparisons, how-tos, and roundups on building AI agents'), ...libraryPosts.map((p) => linkLine(p.title, p.canonical, p.description)), ]), diff --git a/apps/sim/lib/blog/seo.ts b/apps/sim/lib/blog/seo.ts index 86ef6313eea..0ca9a6d779f 100644 --- a/apps/sim/lib/blog/seo.ts +++ b/apps/sim/lib/blog/seo.ts @@ -17,6 +17,7 @@ export const BLOG_SECTION: ContentSection = { name: 'Blog', basePath: '/blog', feedPath: '/blog/rss.xml', + hasAuthorPages: true, description: 'Announcements, insights, and guides for building AI agents.', } diff --git a/apps/sim/lib/content/faq.tsx b/apps/sim/lib/content/faq.tsx index 327440ae740..fa33b514576 100644 --- a/apps/sim/lib/content/faq.tsx +++ b/apps/sim/lib/content/faq.tsx @@ -1,19 +1,13 @@ export function FAQ({ items }: { items: { q: string; a: string }[] }) { if (!items || items.length === 0) return null return ( -
+

FAQ

{items.map((it, i) => ( -
-

- {it.q} -

-
-

- {it.a} -

-
+
+

{it.q}

+

{it.a}

))}
diff --git a/apps/sim/lib/content/seo.ts b/apps/sim/lib/content/seo.ts index 785f2beb565..7d0d01b4676 100644 --- a/apps/sim/lib/content/seo.ts +++ b/apps/sim/lib/content/seo.ts @@ -19,11 +19,39 @@ export interface ContentSection { description: string /** Visible article text available for spoken summaries; defaults to headline and description. */ speakableSelectors?: readonly string[] - /** RSS feed path advertised by the index page's ``, e.g. "/blog/rss.xml". */ + /** RSS feed path advertised by the index and post pages' ``, e.g. "/blog/rss.xml". */ feedPath?: string + /** Whether `/authors/` profile pages exist, so post authors can link to them. */ + hasAuthorPages?: boolean } -export function buildPostMetadata(post: ContentMeta): Metadata { +/** The `speakable` selectors for the `data-speakable` hooks on the post header. */ +const POST_SPEAKABLE_SELECTORS = ['[data-speakable="headline"]', '[data-speakable="description"]'] + +function authorPageUrl(section: ContentSection, id: string): string { + return `${SITE_URL}${section.basePath}/authors/${encodeURIComponent(id)}` +} + +/** + * The `@id` every `Person` node for an author shares, so a post's author and the author's + * profile page resolve to one entity. + */ +function authorEntityId(section: ContentSection, id: string): string { + return `${authorPageUrl(section, id)}#person` +} + +function buildAuthorJsonLd(author: Author, section: ContentSection) { + return { + '@type': 'Person', + ...(section.hasAuthorPages + ? { '@id': authorEntityId(section, author.id), url: authorPageUrl(section, author.id) } + : { url: author.url }), + name: author.name, + ...(author.url ? { sameAs: [author.url] } : {}), + } +} + +export function buildPostMetadata(post: ContentMeta, section: ContentSection): Metadata { const base = new URL(post.canonical) const baseUrl = `${base.protocol}//${base.host}` const searchTitle = post.seoTitle ?? post.title @@ -41,7 +69,12 @@ export function buildPostMetadata(post: ContentMeta): Metadata { robots: post.draft ? { index: false, follow: false, googleBot: { index: false, follow: false } } : { index: true, follow: true, googleBot: { index: true, follow: true } }, - alternates: { canonical: post.canonical }, + alternates: { + canonical: post.canonical, + types: { + ...(section.feedPath ? { 'application/rss+xml': `${SITE_URL}${section.feedPath}` } : {}), + }, + }, openGraph: { title: searchTitle, description: searchDescription, @@ -90,12 +123,10 @@ export function buildPostMetadata(post: ContentMeta): Metadata { * posts that are genuinely technical/developer content (`post.technical`) — * general announcements (funding, company news) get `BlogPosting` alone. */ -export function buildArticleJsonLd( - post: ContentMeta, - speakableSelectors: readonly string[] = ['[itemprop="headline"]', '[itemprop="description"]'] -) { +export function buildArticleJsonLd(post: ContentMeta, section: ContentSection) { return { '@type': post.technical ? ['BlogPosting', 'TechArticle'] : 'BlogPosting', + '@id': `${post.canonical}#article`, url: post.canonical, headline: post.title, description: post.description, @@ -112,12 +143,9 @@ export function buildArticleJsonLd( dateModified: post.updated ?? post.date, wordCount: post.wordCount, ...(post.technical ? { proficiencyLevel: 'Beginner' } : {}), - author: (post.authors && post.authors.length > 0 ? post.authors : [post.author]).map((a) => ({ - '@type': 'Person', - name: a.name, - url: a.url, - ...(a.url ? { sameAs: [a.url] } : {}), - })), + author: (post.authors && post.authors.length > 0 ? post.authors : [post.author]).map((a) => + buildAuthorJsonLd(a, section) + ), publisher: { '@type': 'Organization', '@id': `${SITE_URL}#organization`, @@ -132,6 +160,7 @@ export function buildArticleJsonLd( '@type': 'WebPage', '@id': post.canonical, }, + isPartOf: { '@id': `${SITE_URL}#website` }, keywords: post.tags.join(', '), about: (post.about || []).map((a) => ({ '@type': 'Thing', name: a })), isAccessibleForFree: true, @@ -140,7 +169,7 @@ export function buildArticleJsonLd( inLanguage: 'en-US', speakable: { '@type': 'SpeakableSpecification', - cssSelector: speakableSelectors, + cssSelector: section.speakableSelectors ?? POST_SPEAKABLE_SELECTORS, }, } } @@ -175,7 +204,7 @@ export function buildFaqJsonLd(items: { q: string; a: string }[] | undefined) { export function buildPostGraphJsonLd(post: ContentMeta, section: ContentSection) { const graph: Record[] = [ - buildArticleJsonLd(post, section.speakableSelectors), + buildArticleJsonLd(post, section), buildBreadcrumbJsonLd(post, section), ] @@ -302,7 +331,7 @@ export function buildAuthorMetadata( author?: Author ): Metadata { const name = author?.name ?? 'Author' - const canonical = `${SITE_URL}${section.basePath}/authors/${encodeURIComponent(id)}` + const canonical = authorPageUrl(section, id) const description = `Read articles by ${name} on the Sim ${section.name.toLowerCase()}.` return { title: `${name} | ${section.name}`, @@ -332,8 +361,9 @@ export function buildAuthorGraphJsonLd(section: ContentSection, author: Author) '@graph': [ { '@type': 'Person', + '@id': authorEntityId(section, author.id), name: author.name, - url: `${SITE_URL}${section.basePath}/authors/${encodeURIComponent(author.id)}`, + url: authorPageUrl(section, author.id), sameAs: author.url ? [author.url] : [], image: author.avatarUrl?.startsWith('http') ? author.avatarUrl @@ -358,7 +388,7 @@ export function buildAuthorGraphJsonLd(section: ContentSection, author: Author) '@type': 'ListItem', position: 3, name: author.name, - item: `${SITE_URL}${section.basePath}/authors/${encodeURIComponent(author.id)}`, + item: authorPageUrl(section, author.id), }, ], }, diff --git a/apps/sim/lib/library/seo.ts b/apps/sim/lib/library/seo.ts index 2644d258578..ceb15b8d087 100644 --- a/apps/sim/lib/library/seo.ts +++ b/apps/sim/lib/library/seo.ts @@ -1,3 +1,4 @@ +import type { Metadata } from 'next' import type { Author, ContentMeta } from '@/lib/content/schema' import type { ContentSection } from '@/lib/content/seo' import { @@ -8,7 +9,7 @@ import { buildFaqJsonLd, buildIndexMetadata as buildIndexMetadataGeneric, buildPostGraphJsonLd as buildPostGraphJsonLdGeneric, - buildPostMetadata, + buildPostMetadata as buildPostMetadataGeneric, buildTagsBreadcrumbJsonLd as buildTagsBreadcrumbJsonLdGeneric, buildTagsMetadata as buildTagsMetadataGeneric, } from '@/lib/content/seo' @@ -17,11 +18,24 @@ export const LIBRARY_SECTION: ContentSection = { name: 'Library', basePath: '/library', feedPath: '/library/rss.xml', + hasAuthorPages: true, description: 'Comparisons, how-tos, and roundups for teams evaluating and building AI agents with Sim.', } -export { buildArticleJsonLd, buildFaqJsonLd, buildPostMetadata } +export { buildArticleJsonLd, buildFaqJsonLd } + +/** Post metadata plus the `text/markdown` alternate that `app/library.md/[slug]` serves. */ +export function buildPostMetadata(post: ContentMeta): Metadata { + const metadata = buildPostMetadataGeneric(post, LIBRARY_SECTION) + return { + ...metadata, + alternates: { + ...metadata.alternates, + types: { ...metadata.alternates?.types, 'text/markdown': `${post.canonical}.md` }, + }, + } +} export function buildPostGraphJsonLd(post: ContentMeta) { return buildPostGraphJsonLdGeneric(post, LIBRARY_SECTION) diff --git a/apps/sim/next.config.ts b/apps/sim/next.config.ts index dfd78ed898a..66a6a65ca8a 100644 --- a/apps/sim/next.config.ts +++ b/apps/sim/next.config.ts @@ -283,6 +283,11 @@ const nextConfig: NextConfig = { source: '/', headers: [{ key: 'Vary', value: 'Accept' }], }, + { + /** A library article answers in HTML or Markdown by `Accept` (see the `afterFiles` rewrites). */ + source: '/library/:slug', + headers: [{ key: 'Vary', value: 'Accept' }], + }, { // `/public`-served assets keep their path across deploys (no content // hash), so a shorter TTL + revalidation window bounds how long a @@ -690,7 +695,13 @@ const nextConfig: NextConfig = { return { /** `beforeFiles` so a Markdown client's `/` reaches `/index.md` ahead of the HTML page. */ beforeFiles: [{ source: '/', ...MARKDOWN_REQUEST, destination: '/index.md' }], + /** + * `afterFiles` so `/library/tags`, the feeds, and every other static library route keep + * their own responses; only the dynamic article route is shadowed. + */ afterFiles: [ + { source: '/library/:slug.md', destination: '/library.md/:slug' }, + { source: '/library/:slug', ...MARKDOWN_REQUEST, destination: '/library.md/:slug' }, { source: '/favicon.ico', destination: '/icon.svg', diff --git a/apps/sim/public/e300fec30f4d245dc5ad832693bbc144.txt b/apps/sim/public/e300fec30f4d245dc5ad832693bbc144.txt new file mode 100644 index 00000000000..b95d5297462 --- /dev/null +++ b/apps/sim/public/e300fec30f4d245dc5ad832693bbc144.txt @@ -0,0 +1 @@ +e300fec30f4d245dc5ad832693bbc144 \ No newline at end of file diff --git a/apps/sim/scripts/test-landing-seo-e2e.ts b/apps/sim/scripts/test-landing-seo-e2e.ts index 32a05a06ddd..3903fa51d3c 100644 --- a/apps/sim/scripts/test-landing-seo-e2e.ts +++ b/apps/sim/scripts/test-landing-seo-e2e.ts @@ -48,6 +48,7 @@ interface Integration { updatedAt?: string } const integrations = integrationsJson.integrations as Integration[] +const LIBRARY_ARTICLE_PATH = /^\/library\/(?!authors\/|tags)[^/]+$/ /** `?home` serves the landing page without the self-hosted `/` → `/login` redirect. */ const HOME_PATH = '/?home' @@ -196,7 +197,7 @@ async function run() { '/blog', '/library', firstPath(/^\/blog\/(?!authors\/|tags)[^/]+$/), - firstPath(/^\/library\/(?!authors\/|tags)[^/]+$/), + firstPath(LIBRARY_ARTICLE_PATH), firstPath(/^\/integrations\/[^/]+$/), firstPath(/^\/models\/anthropic\/[^/]+$/), '/models/context-windows', @@ -481,6 +482,72 @@ async function run() { assert.equal(direct.body, markdown.body) assert.match(direct.headers.get('link') ?? '', /rel="canonical"/) }) + + await check('a library article is served as Markdown at .md and by Accept', async () => { + const path = firstPath(LIBRARY_ARTICLE_PATH) + const html = await get(path, 'text/html,application/xhtml+xml,*/*;q=0.8') + assert.match(html.headers.get('content-type') ?? '', /^text\/html/) + assert.match(html.headers.get('vary') ?? '', /\bAccept\b/) + const alternate = /]*>([^<]*)<\/h1>/.exec(html.body)?.[1] ?? '') + assert.equal(direct.body.split('\n')[0], `# ${heading}`) + + const negotiated = await get(path, 'text/markdown') + assert.match(negotiated.headers.get('content-type') ?? '', /^text\/markdown/) + assert.equal(negotiated.body, direct.body) + + const tags = await get('/library/tags', 'text/markdown, text/html;q=0.5') + assert.equal(tags.status, 200, 'the Markdown rewrite shadows /library/tags') + assert.match(tags.headers.get('content-type') ?? '', /^text\/html/) + + const unknown = await get('/library/not-a-real-post.md') + assert.equal(unknown.status, 404) + assert.match(unknown.headers.get('content-type') ?? '', /^text\/markdown/) + }) + + await check( + 'a library article has one structured-data source, linked to the site graph', + async () => { + const path = firstPath(LIBRARY_ARTICLE_PATH) + const { body } = await get(path) + assert.doesNotMatch(body, /itemType="https:\/\/schema\.org\/(BlogPosting|FAQPage)"/i) + const nodes = jsonLdNodes(body) + const website = nodes.find((node) => node['@type'] === 'WebSite') + const article = nodes.find((node) => [node['@type']].flat().includes('BlogPosting')) + assert(website && article, `${path} lacks a WebSite or BlogPosting node`) + assert.equal(article['@id'], `${article.url}#article`) + for (const selector of (article.speakable as { cssSelector: string[] }).cssSelector) { + const [, name, value] = /^\[([\w-]+)="([^"]+)"\]$/.exec(selector) ?? [] + assert( + body.includes(`${name}="${value}"`), + `speakable selector ${selector} matches nothing` + ) + } + assert.equal( + (article.isPartOf as Record | undefined)?.['@id'], + website['@id'] + ) + for (const author of article.author as Record[]) { + const page = new URL(String(author.url)).pathname + assert.match(page, /^\/library\/authors\/[^/]+$/) + assert.equal(author['@id'], `${author.url}#person`) + const profile = jsonLdNodes((await get(page)).body).find( + (node) => node['@type'] === 'Person' + ) + assert.equal(profile?.['@id'], author['@id'], `${page} names a different Person @id`) + } + } + ) } try { diff --git a/scripts/ping-indexnow.test.ts b/scripts/ping-indexnow.test.ts new file mode 100644 index 00000000000..b106584720a --- /dev/null +++ b/scripts/ping-indexnow.test.ts @@ -0,0 +1,113 @@ +import { spawnSync } from 'node:child_process' +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { collectIndexNowUrls } from './ping-indexnow' + +const LIBRARY = 'apps/sim/content/library' +const COMPETITORS = 'apps/sim/lib/compare/data/competitors' +const repos: string[] = [] + +function git(cwd: string, ...args: string[]): string { + const result = spawnSync('git', args, { cwd, encoding: 'utf8' }) + if (result.status !== 0) throw new Error(`git ${args.join(' ')}: ${result.stderr}`) + return result.stdout.trim() +} + +function write(repo: string, file: string, content: string) { + mkdirSync(dirname(join(repo, file)), { recursive: true }) + writeFileSync(join(repo, file), content) +} + +function post(draft = false): string { + return `---\nslug: x\ntitle: A post\ndraft: ${draft}\n---\n\nBody\n` +} + +function commit(repo: string): string { + git(repo, 'add', '-A') + git( + repo, + '-c', + 'user.name=t', + '-c', + 'user.email=t@t', + '-c', + 'commit.gpgsign=false', + 'commit', + '-qm', + 'c', + '--allow-empty' + ) + return git(repo, 'rev-parse', 'HEAD') +} + +/** A real repository with one published post, one draft, and two competitors at its base commit. */ +function fixture() { + const repo = mkdtempSync(join(tmpdir(), 'indexnow-')) + repos.push(repo) + git(repo, 'init', '-q') + write(repo, `${LIBRARY}/published/index.mdx`, post()) + write(repo, `${LIBRARY}/hidden/index.mdx`, post(true)) + write(repo, `${COMPETITORS}/n8n.ts`, "export const p = {\n id: 'n8n',\n}\n") + write(repo, `${COMPETITORS}/stackai.ts`, "export const p = {\n id: 'stack-ai',\n}\n") + write(repo, 'apps/sim/lib/compare/data/sim.ts', 'export const sim = 1\n') + write(repo, 'README.md', 'readme\n') + const base = commit(repo) + return { repo, base, urls: () => collectIndexNowUrls(repo, base, commit(repo)) } +} + +afterEach(() => { + for (const repo of repos.splice(0)) rmSync(repo, { recursive: true, force: true }) +}) + +describe('collectIndexNowUrls', () => { + it('submits an edited published post and the library index', () => { + const { repo, urls } = fixture() + write(repo, `${LIBRARY}/published/cover.jpg`, 'new image') + expect(urls()).toEqual(['https://www.sim.ai/library', 'https://www.sim.ai/library/published']) + }) + + it('never submits a draft', () => { + const { repo, urls } = fixture() + write(repo, `${LIBRARY}/hidden/index.mdx`, `${post(true)}more\n`) + expect(urls()).toEqual([]) + }) + + it('submits a renamed post under its old and new slug, and never a deleted draft', () => { + const { repo, urls } = fixture() + git(repo, 'mv', `${LIBRARY}/published`, `${LIBRARY}/renamed`) + git(repo, 'rm', '-rq', `${LIBRARY}/hidden`) + expect(urls()).toEqual([ + 'https://www.sim.ai/library', + 'https://www.sim.ai/library/published', + 'https://www.sim.ai/library/renamed', + ]) + }) + + it('names a comparison by its profile id, not its file name', () => { + const { repo, urls } = fixture() + write(repo, `${COMPETITORS}/stackai.ts`, "export const p = {\n id: 'stack-ai',\n x: 1,\n}\n") + expect(urls()).toEqual([ + 'https://www.sim.ai/comparisons', + 'https://www.sim.ai/comparisons/stack-ai', + ]) + }) + + it("submits every comparison when Sim's own profile changes", () => { + const { repo, urls } = fixture() + write(repo, 'apps/sim/lib/compare/data/sim.ts', 'export const sim = 2\n') + expect(urls()).toEqual([ + 'https://www.sim.ai/comparisons', + 'https://www.sim.ai/comparisons/n8n', + 'https://www.sim.ai/comparisons/stack-ai', + ]) + }) + + it('submits nothing when no library or comparison source changed', () => { + const { repo, urls } = fixture() + write(repo, 'README.md', 'changed\n') + write(repo, `${LIBRARY}/notes.md`, 'not a post\n') + expect(urls()).toEqual([]) + }) +}) diff --git a/scripts/ping-indexnow.ts b/scripts/ping-indexnow.ts new file mode 100644 index 00000000000..c1640ab03ec --- /dev/null +++ b/scripts/ping-indexnow.ts @@ -0,0 +1,118 @@ +#!/usr/bin/env bun +/** + * Tells IndexNow (Bing, Yandex, Seznam, Naver; ChatGPT and Copilot search ground on Bing) which + * library articles and comparison pages a production release changed, so they are recrawled + * without waiting for the sitemap. CI runs it after the production cutover, with the commits on + * either side of the push: `bun scripts/ping-indexnow.ts `. + * + * Best effort by design: the CI job continues on error, and a run with nothing to submit, or + * without a base commit to diff against, exits without a request. + * + * The key is public (IndexNow verifies ownership by fetching it from the site), so it is committed + * as `apps/sim/public/.txt`. + */ +import { spawnSync } from 'node:child_process' +import path from 'node:path' +import { SIM_SITE_URL } from '../packages/utils/src/site' + +const SITE = SIM_SITE_URL +const INDEXNOW_KEY = 'e300fec30f4d245dc5ad832693bbc144' +const INDEXNOW_ENDPOINT = 'https://api.indexnow.org/indexnow' +/** IndexNow accepts at most 10,000 URLs per request. */ +const MAX_URLS = 10_000 +const REQUEST_TIMEOUT_MS = 15_000 + +const LIBRARY_DIR = 'apps/sim/content/library' +const COMPETITORS_DIR = 'apps/sim/lib/compare/data/competitors' +const SIM_PROFILE = 'apps/sim/lib/compare/data/sim.ts' + +function git(repo: string, args: string[]): string | null { + const result = spawnSync('git', args, { + cwd: repo, + encoding: 'utf8', + maxBuffer: 64 * 1024 * 1024, + }) + return result.status === 0 ? result.stdout : null +} + +function isDraft(source: string): boolean { + const frontmatter = /^---\r?\n([\s\S]*?)\r?\n---/.exec(source)?.[1] ?? '' + return /^draft:\s*true\s*(#.*)?\r?$/m.test(frontmatter) +} + +function profileId(source: string): string | undefined { + return /^\s*id:\s*['"]([^'"]+)['"]/m.exec(source)?.[1] +} + +/** + * The public URLs whose source changed between two commits: each published library article + * (renamed and deleted ones under their old slug too, so engines learn of the redirect or 404), + * each comparison whose profile changed (every comparison when Sim's own profile did), and the + * index page of each section that changed. Drafts are never submitted. + */ +export function collectIndexNowUrls(repo: string, base: string, head: string): string[] { + const diff = git(repo, ['diff', '--name-status', '-M', base, head, '--']) ?? '' + const show = (sha: string, file: string) => git(repo, ['show', `${sha}:${file}`]) + const paths = diff + .split('\n') + .filter(Boolean) + .flatMap((line) => line.split('\t').slice(1)) + const urls = new Set() + + const slugs = new Set( + paths + .map((file) => path.relative(LIBRARY_DIR, file).split('/')) + .filter((parts) => parts.length > 1 && parts[0] !== '..') + .map(([slug]) => slug) + ) + for (const slug of slugs) { + const index = `${LIBRARY_DIR}/${slug}/index.mdx` + const source = show(head, index) ?? show(base, index) + if (source !== null && !isDraft(source)) urls.add(`${SITE}/library/${slug}`) + } + if (urls.size > 0) urls.add(`${SITE}/library`) + + const profiles = paths.includes(SIM_PROFILE) + ? (git(repo, ['ls-tree', '--name-only', head, `${COMPETITORS_DIR}/`]) ?? '') + .split('\n') + .filter(Boolean) + : paths.filter((file) => path.dirname(file) === COMPETITORS_DIR && file.endsWith('.ts')) + const ids = profiles + .map((file) => profileId(show(head, file) ?? show(base, file) ?? '')) + .filter(Boolean) + for (const id of ids) urls.add(`${SITE}/comparisons/${id}`) + if (ids.length > 0) urls.add(`${SITE}/comparisons`) + + return [...urls].sort() +} + +async function main() { + const [base, head = 'HEAD'] = process.argv.slice(2) + const repo = path.resolve(import.meta.dir, '..') + if (!base || git(repo, ['cat-file', '-e', `${base}^{commit}`]) === null) { + console.info(`No base commit to diff against (${base || 'none'}); nothing submitted.`) + return + } + + const urls = collectIndexNowUrls(repo, base, head).slice(0, MAX_URLS) + if (urls.length === 0) { + console.info('No library or comparison page changed; nothing submitted.') + return + } + + const response = await fetch(INDEXNOW_ENDPOINT, { + method: 'POST', + headers: { 'Content-Type': 'application/json; charset=utf-8' }, + body: JSON.stringify({ + host: new URL(SITE).host, + key: INDEXNOW_KEY, + keyLocation: `${SITE}/${INDEXNOW_KEY}.txt`, + urlList: urls, + }), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }) + console.info(`IndexNow answered ${response.status} for ${urls.length} URLs:\n${urls.join('\n')}`) + if (!response.ok) process.exitCode = 1 +} + +if (import.meta.main) await main() From c493cbef4e49bab072837368c833a770361306d2 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 23 Oct 2026 16:16:22 -0700 Subject: [PATCH 2/5] improvement(library): link Andrew's author profile and surface the docs history fetch outcome in build logs --- apps/docs/scripts/fetch-git-history.ts | 5 ++++- apps/sim/content/authors/andrew.json | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/docs/scripts/fetch-git-history.ts b/apps/docs/scripts/fetch-git-history.ts index dd87c8f8012..cefc9d847ea 100644 --- a/apps/docs/scripts/fetch-git-history.ts +++ b/apps/docs/scripts/fetch-git-history.ts @@ -11,7 +11,8 @@ import { execFileSync } from 'node:child_process' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -const logger = createLogger('DocsGitHistory') +/** A build step's outcome belongs in the build log, which the production default (errors only) would hide. */ +const logger = createLogger('DocsGitHistory', { logLevel: 'INFO' }) /** Bounds the fetch so a stalled remote costs at most this much of the deployment, never all of it. */ const FETCH_TIMEOUT_MS = 120_000 @@ -29,6 +30,8 @@ if (process.env.VERCEL) { if (git(['rev-parse', '--is-shallow-repository']) === 'true') { git(['fetch', '--unshallow', '--filter=blob:none', '--no-tags', 'origin'], FETCH_TIMEOUT_MS) logger.info('Fetched full commit history for docs last-modified dates') + } else { + logger.info('Clone already has full commit history') } } catch (error) { logger.warn( diff --git a/apps/sim/content/authors/andrew.json b/apps/sim/content/authors/andrew.json index 3b108522492..a2e5dee2960 100644 --- a/apps/sim/content/authors/andrew.json +++ b/apps/sim/content/authors/andrew.json @@ -1,5 +1,6 @@ { "id": "andrew", "name": "Andrew Caslow", + "url": "https://www.linkedin.com/in/andrewcaslow/", "avatarUrl": "/authors/andrew.jpg" } From 7e8a98e029b28e82944263d2df50c0cc32d6b416 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 23 Oct 2026 16:18:50 -0700 Subject: [PATCH 3/5] fix(library): gate IndexNow on the ECS cutover, cover unpublished and renamed pages, and give customer stories speakable hooks --- .github/workflows/ci.yml | 6 ++- .../customer-story-page.tsx | 10 ++++- scripts/ping-indexnow.test.ts | 27 +++++++++++++ scripts/ping-indexnow.ts | 38 +++++++++++-------- 4 files changed, 63 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b49a98311c2..4fc436172e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -569,6 +569,9 @@ jobs: needs.promote.result == 'success' && needs.trigger-upload.result == 'success' && needs.promote.outputs.promoted == 'true' + outputs: + # The new app serves traffic; indexnow keys off this, not the later Trigger.dev promotion. + cutover: ${{ steps.cutover.outcome }} # Mostly waiting on the ECS cutover after a dependency install: the smallest runner is enough. runs-on: *runner-2vcpu # Leave setup/promotion headroom above the 70-minute cutover poll. @@ -596,6 +599,7 @@ jobs: # An unchanged tag may belong to a failed or still-running earlier deploy. # Verify its latest cutover rather than treating tag equality as success. - name: Wait for ECS traffic cutover + id: cutover env: OVERALL_TIMEOUT: 4200 APP_IMAGE_CHANGED: ${{ needs.promote.outputs.app_image_changed }} @@ -646,7 +650,7 @@ jobs: github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.promote.outputs.promoted == 'true' && - needs.trigger-promote.result == 'success' + needs.trigger-promote.outputs.cutover == 'success' runs-on: *runner-2vcpu timeout-minutes: 5 continue-on-error: true diff --git a/apps/sim/app/(landing)/customers/components/customer-story-page/customer-story-page.tsx b/apps/sim/app/(landing)/customers/components/customer-story-page/customer-story-page.tsx index c22930d5d47..4a070b82f1a 100644 --- a/apps/sim/app/(landing)/customers/components/customer-story-page/customer-story-page.tsx +++ b/apps/sim/app/(landing)/customers/components/customer-story-page/customer-story-page.tsx @@ -28,7 +28,10 @@ export function CustomerStoryPage({ story, post, nextStory }: CustomerStoryPageP
-

+

{post.title}

@@ -72,7 +75,10 @@ export function CustomerStoryPage({ story, post, nextStory }: CustomerStoryPageP
-

+

{post.description}

diff --git a/scripts/ping-indexnow.test.ts b/scripts/ping-indexnow.test.ts index b106584720a..40adfe856ad 100644 --- a/scripts/ping-indexnow.test.ts +++ b/scripts/ping-indexnow.test.ts @@ -104,6 +104,33 @@ describe('collectIndexNowUrls', () => { ]) }) + it('submits a post that went from published to draft, since it now 404s', () => { + const { repo, urls } = fixture() + write(repo, `${LIBRARY}/published/index.mdx`, post(true)) + expect(urls()).toEqual(['https://www.sim.ai/library', 'https://www.sim.ai/library/published']) + }) + + it('submits both the old and new URL when a comparison id changes', () => { + const { repo, urls } = fixture() + write(repo, `${COMPETITORS}/n8n.ts`, "export const p = {\n id: 'n8n-io',\n}\n") + expect(urls()).toEqual([ + 'https://www.sim.ai/comparisons', + 'https://www.sim.ai/comparisons/n8n', + 'https://www.sim.ai/comparisons/n8n-io', + ]) + }) + + it("keeps a deleted comparison when Sim's profile changes in the same release", () => { + const { repo, urls } = fixture() + write(repo, 'apps/sim/lib/compare/data/sim.ts', 'export const sim = 2\n') + git(repo, 'rm', '-q', `${COMPETITORS}/n8n.ts`) + expect(urls()).toEqual([ + 'https://www.sim.ai/comparisons', + 'https://www.sim.ai/comparisons/n8n', + 'https://www.sim.ai/comparisons/stack-ai', + ]) + }) + it('submits nothing when no library or comparison source changed', () => { const { repo, urls } = fixture() write(repo, 'README.md', 'changed\n') diff --git a/scripts/ping-indexnow.ts b/scripts/ping-indexnow.ts index c1640ab03ec..d6bc1634fb5 100644 --- a/scripts/ping-indexnow.ts +++ b/scripts/ping-indexnow.ts @@ -45,10 +45,11 @@ function profileId(source: string): string | undefined { } /** - * The public URLs whose source changed between two commits: each published library article - * (renamed and deleted ones under their old slug too, so engines learn of the redirect or 404), - * each comparison whose profile changed (every comparison when Sim's own profile did), and the - * index page of each section that changed. Drafts are never submitted. + * The public URLs whose source changed between two commits: each library article published on + * either side (renamed, deleted, and newly drafted ones too, so engines learn of the 404), each + * comparison whose profile changed under its old and new id (every comparison when Sim's own + * profile did), and the index page of each section that changed. A post that was never published + * is never submitted. */ export function collectIndexNowUrls(repo: string, base: string, head: string): string[] { const diff = git(repo, ['diff', '--name-status', '-M', base, head, '--']) ?? '' @@ -67,21 +68,28 @@ export function collectIndexNowUrls(repo: string, base: string, head: string): s ) for (const slug of slugs) { const index = `${LIBRARY_DIR}/${slug}/index.mdx` - const source = show(head, index) ?? show(base, index) - if (source !== null && !isDraft(source)) urls.add(`${SITE}/library/${slug}`) + const wasOrIsPublished = [show(base, index), show(head, index)].some( + (source) => source !== null && !isDraft(source) + ) + if (wasOrIsPublished) urls.add(`${SITE}/library/${slug}`) } if (urls.size > 0) urls.add(`${SITE}/library`) - const profiles = paths.includes(SIM_PROFILE) - ? (git(repo, ['ls-tree', '--name-only', head, `${COMPETITORS_DIR}/`]) ?? '') - .split('\n') - .filter(Boolean) - : paths.filter((file) => path.dirname(file) === COMPETITORS_DIR && file.endsWith('.ts')) - const ids = profiles - .map((file) => profileId(show(head, file) ?? show(base, file) ?? '')) - .filter(Boolean) + const changedProfiles = paths.filter( + (file) => path.dirname(file) === COMPETITORS_DIR && file.endsWith('.ts') + ) + const currentProfiles = paths.includes(SIM_PROFILE) + ? (git(repo, ['ls-tree', '--name-only', head, `${COMPETITORS_DIR}/`]) ?? '').split('\n') + : [] + const ids = new Set( + [...new Set([...changedProfiles, ...currentProfiles])] + .filter(Boolean) + .flatMap((file) => [show(base, file), show(head, file)]) + .map((source) => profileId(source ?? '')) + .filter((id): id is string => Boolean(id)) + ) for (const id of ids) urls.add(`${SITE}/comparisons/${id}`) - if (ids.length > 0) urls.add(`${SITE}/comparisons`) + if (ids.size > 0) urls.add(`${SITE}/comparisons`) return [...urls].sort() } From 65564a5accd2daff7d46c49b8d53a22d1c43c29c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 23 Oct 2026 16:21:14 -0700 Subject: [PATCH 4/5] fix(docs): fetch commit history from the repository Vercel names, since its clone has no origin remote --- apps/docs/scripts/fetch-git-history.ts | 33 ++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/apps/docs/scripts/fetch-git-history.ts b/apps/docs/scripts/fetch-git-history.ts index cefc9d847ea..0eb33efa089 100644 --- a/apps/docs/scripts/fetch-git-history.ts +++ b/apps/docs/scripts/fetch-git-history.ts @@ -3,6 +3,8 @@ * shallow clone (every file untouched since the boundary commit would get that commit's date), so * the docs sitemap shipped with no `lastModified` at all. Fetch the commit history before the * build. `--filter=blob:none` skips file contents: `git log --name-only` needs commits and trees. + * Vercel's clone has no `origin` remote, so the history comes from the public repository URL its + * system environment names, pinned to the commit being built. * * A failed fetch leaves the clone shallow, so pages ship undated as before. That is logged rather * than failing the deploy, because dates are an enhancement and the docs must still ship. @@ -25,10 +27,37 @@ function git(args: string[], timeout?: number): string { }).trim() } -if (process.env.VERCEL) { +const { + VERCEL, + VERCEL_GIT_PROVIDER, + VERCEL_GIT_REPO_OWNER, + VERCEL_GIT_REPO_SLUG, + VERCEL_GIT_COMMIT_SHA, +} = process.env + +if (VERCEL) { try { if (git(['rev-parse', '--is-shallow-repository']) === 'true') { - git(['fetch', '--unshallow', '--filter=blob:none', '--no-tags', 'origin'], FETCH_TIMEOUT_MS) + if ( + VERCEL_GIT_PROVIDER !== 'github' || + !VERCEL_GIT_REPO_OWNER || + !VERCEL_GIT_REPO_SLUG || + !VERCEL_GIT_COMMIT_SHA + ) { + throw new Error('Vercel did not provide the GitHub repository and commit for this build') + } + const repositoryUrl = `https://github.com/${VERCEL_GIT_REPO_OWNER}/${VERCEL_GIT_REPO_SLUG}.git` + git( + [ + 'fetch', + '--unshallow', + '--filter=blob:none', + '--no-tags', + repositoryUrl, + VERCEL_GIT_COMMIT_SHA, + ], + FETCH_TIMEOUT_MS + ) logger.info('Fetched full commit history for docs last-modified dates') } else { logger.info('Clone already has full commit history') From c22d4145da7cdfb1e45e1c096bcc4efa79935b01 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 23 Oct 2026 16:33:21 -0700 Subject: [PATCH 5/5] fix(library): noindex direct Markdown article URLs on self-hosted deployments and diff IndexNow from the last submitted revision --- .github/workflows/ci.yml | 23 ++++++++++++++++++++--- apps/sim/lib/landing/routes.ts | 9 ++++++++- apps/sim/proxy.test.ts | 3 +++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4fc436172e6..18ccd11c62f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -656,6 +656,7 @@ jobs: continue-on-error: true permissions: contents: read + actions: read steps: - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -665,12 +666,28 @@ jobs: with: bun-version: 1.4.4 - # The diff needs only the two trees, so the previous head is fetched alone. A push that - # created the branch has an all-zero `before`, which the script skips. + # Diff from the head of the last earlier run whose submission succeeded, not the previous + # push: a push that never deployed, or whose submission failed, would otherwise drop its + # changed URLs for good. Falls back to the previous push when no such run is in view. The + # diff needs only the two trees, so that one commit is fetched alone; an all-zero `before` + # (the branch's first push) has no base, which the script skips. - name: Submit changed URLs to IndexNow env: - BASE_SHA: ${{ github.event.before }} + GH_TOKEN: ${{ github.token }} + PUSH_BEFORE: ${{ github.event.before }} run: | + BASE_SHA="" + while read -r run_id head_sha; do + conclusion="$(gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/jobs?per_page=100" \ + --jq '.jobs[] | select(.name == "indexnow") | .conclusion')" + if [[ "$conclusion" == "success" ]]; then + BASE_SHA="$head_sha" + break + fi + done < <(gh api "repos/$GITHUB_REPOSITORY/actions/workflows/ci.yml/runs?branch=main&event=push&per_page=30" \ + --jq ".workflow_runs[] | select(.id < $GITHUB_RUN_ID) | \"\\(.id) \\(.head_sha)\"") + BASE_SHA="${BASE_SHA:-$PUSH_BEFORE}" + echo "Submitting URLs changed in $BASE_SHA..$GITHUB_SHA" if [[ ! "$BASE_SHA" =~ ^0+$ ]]; then git fetch --no-tags --depth=1 origin "$BASE_SHA" fi diff --git a/apps/sim/lib/landing/routes.ts b/apps/sim/lib/landing/routes.ts index 727081a58eb..199df2211ae 100644 --- a/apps/sim/lib/landing/routes.ts +++ b/apps/sim/lib/landing/routes.ts @@ -70,7 +70,14 @@ const MARKETING_DOCUMENT_PATHS: ReadonlySet = new Set([ '/changelog.xml', ]) +/** Top-level route segments that republish landing content as documents, e.g. `/library.md/`. */ +const MARKETING_DOCUMENT_SEGMENTS: ReadonlySet = new Set(['library.md']) + /** Whether a pathname serves marketing content: a landing page or a marketing document. */ export function isMarketingPath(pathname: string): boolean { - return isLandingPath(pathname) || MARKETING_DOCUMENT_PATHS.has(pathname) + return ( + isLandingPath(pathname) || + MARKETING_DOCUMENT_PATHS.has(pathname) || + MARKETING_DOCUMENT_SEGMENTS.has(pathname.split('/')[1]) + ) } diff --git a/apps/sim/proxy.test.ts b/apps/sim/proxy.test.ts index a89b63c78b3..12f033518e6 100644 --- a/apps/sim/proxy.test.ts +++ b/apps/sim/proxy.test.ts @@ -179,6 +179,7 @@ describe('proxy indexing policy', () => { expect(robotsTag('/llms.txt')).toBe('noindex, nofollow') expect(robotsTag('/llms-full.txt')).toBe('noindex, nofollow') expect(robotsTag('/changelog.xml')).toBe('noindex, nofollow') + expect(robotsTag('/library.md/some-article')).toBe('noindex, nofollow') }) it('leaves the marketing pages and documents of the hosted deployment indexable', () => { @@ -189,6 +190,7 @@ describe('proxy indexing policy', () => { expect(robotsTag('/llms.txt')).toBeNull() expect(robotsTag('/llms-full.txt')).toBeNull() expect(robotsTag('/changelog.xml')).toBeNull() + expect(robotsTag('/library.md/some-article')).toBeNull() }) it('leaves a self-hosted non-landing public page indexable', () => { @@ -216,6 +218,7 @@ describe('proxy matcher', () => { expect(matches('/llms.txt')).toBe(true) expect(matches('/llms-full.txt')).toBe(true) expect(matches('/changelog.xml')).toBe(true) + expect(matches('/library.md/some-article')).toBe(true) }) it('still runs for every other API route', () => {