From ac104ae1cc8b482136bc52addbf7ce124218ed3f Mon Sep 17 00:00:00 2001 From: capJavert Date: Wed, 22 Jul 2026 14:40:35 +0200 Subject: [PATCH 1/5] feat: web search and notification adjustment --- .../common/interest/discoverExternalUrls.ts | 86 +++++++++++ __tests__/common/interest/runInterestAgent.ts | 36 +++++ __tests__/notifications/index.ts | 57 ++++++++ src/common/interest/discoverExternalUrls.ts | 125 ++++++++++++++++ src/common/interest/runInterestAgent.ts | 137 +++++++++++++++++- src/entity/InterestFinding.ts | 9 ++ .../1784719854693-AddInterestFindingOrigin.ts | 17 +++ src/notifications/generate.ts | 2 +- src/remoteConfig.ts | 2 + .../interest/postVisibleInterestMatch.ts | 2 + 10 files changed, 467 insertions(+), 6 deletions(-) create mode 100644 __tests__/common/interest/discoverExternalUrls.ts create mode 100644 src/common/interest/discoverExternalUrls.ts create mode 100644 src/migration/1784719854693-AddInterestFindingOrigin.ts diff --git a/__tests__/common/interest/discoverExternalUrls.ts b/__tests__/common/interest/discoverExternalUrls.ts new file mode 100644 index 0000000000..83cd7dc3d9 --- /dev/null +++ b/__tests__/common/interest/discoverExternalUrls.ts @@ -0,0 +1,86 @@ +import type { FastifyBaseLogger } from 'fastify'; +import { anthropicClient } from '../../../src/integrations/anthropic/client'; +import { discoverExternalUrls } from '../../../src/common/interest/discoverExternalUrls'; + +const logger = { + child: () => ({ info: jest.fn(), warn: jest.fn() }), +} as unknown as FastifyBaseLogger; + +const interest = { + id: 'i1', + query: 'cool zig projects', +}; + +const mockContent = (text: string) => + jest + .spyOn(anthropicClient, 'createMessage') + .mockResolvedValue({ content: [{ type: 'text', text }] } as never); + +afterEach(() => { + jest.restoreAllMocks(); +}); + +describe('discoverExternalUrls', () => { + it('parses a JSON array of candidates from the model reply', async () => { + mockContent( + '[{"url":"https://a.com","title":"A","rationale":"r","score":0.9}]', + ); + const result = await discoverExternalUrls({ + interest, + query: 'zig', + logger, + }); + expect(result).toEqual([ + { url: 'https://a.com', title: 'A', rationale: 'r', score: 0.9 }, + ]); + }); + + it('drops non-http entries and de-duplicates urls', async () => { + mockContent( + JSON.stringify([ + { url: 'https://a.com', title: 'A', rationale: 'r', score: 0.8 }, + { url: 'https://a.com', title: 'dup', rationale: 'r', score: 0.8 }, + { url: 'ftp://b.com', title: 'B', rationale: 'r', score: 0.8 }, + { url: 42, title: 'C', rationale: 'r', score: 0.8 }, + ]), + ); + const result = await discoverExternalUrls({ + interest, + query: 'zig', + logger, + }); + expect(result.map((r) => r.url)).toEqual(['https://a.com']); + }); + + it('returns an empty array when the reply is not a JSON array', async () => { + mockContent('I could not find anything relevant.'); + const result = await discoverExternalUrls({ + interest, + query: 'zig', + logger, + }); + expect(result).toEqual([]); + }); + + it('returns an empty array (does not throw) when the client fails', async () => { + jest + .spyOn(anthropicClient, 'createMessage') + .mockRejectedValue(new Error('boom')); + const result = await discoverExternalUrls({ + interest, + query: 'zig', + logger, + }); + expect(result).toEqual([]); + }); + + it('issues a generic web_search tool with no domain restriction', async () => { + const spy = mockContent('[]'); + await discoverExternalUrls({ interest, query: 'zig', logger }); + const request = spy.mock.calls[0][0] as { + tools: Array<{ name?: string; allowed_domains?: string[] }>; + }; + expect(request.tools[0].name).toBe('web_search'); + expect(request.tools[0].allowed_domains).toBeUndefined(); + }); +}); diff --git a/__tests__/common/interest/runInterestAgent.ts b/__tests__/common/interest/runInterestAgent.ts index fd14a26d51..ae831aa3bd 100644 --- a/__tests__/common/interest/runInterestAgent.ts +++ b/__tests__/common/interest/runInterestAgent.ts @@ -33,4 +33,40 @@ describe('getInterestAgentTools', () => { expect(tools).toContain('add_to_feed'); expect(tools).not.toContain('write_post'); }); + + it('omits discover_external when the web source is off', () => { + const tools = getInterestAgentTools(undefined, { + dailyDev: true, + web: false, + github: false, + }); + expect(tools).not.toContain('discover_external'); + }); + + it('includes discover_external when the web source is on', () => { + expect( + getInterestAgentTools(undefined, { + dailyDev: true, + web: true, + github: false, + }), + ).toContain('discover_external'); + }); + + it('does not enable discover_external for the github source alone (reserved for a future github tool)', () => { + const tools = getInterestAgentTools(undefined, { + dailyDev: true, + web: false, + github: true, + }); + expect(tools).not.toContain('discover_external'); + }); + + it('omits discover_external when feed output is off even with web source on', () => { + const tools = getInterestAgentTools( + { feed: false, post: true, digest: false, notification: true }, + { dailyDev: true, web: true, github: false }, + ); + expect(tools).not.toContain('discover_external'); + }); }); diff --git a/__tests__/notifications/index.ts b/__tests__/notifications/index.ts index f8acfcdae5..392fd71e40 100644 --- a/__tests__/notifications/index.ts +++ b/__tests__/notifications/index.ts @@ -1386,6 +1386,63 @@ describe('storeNotificationBundle', () => { expect(avatars.length).toEqual(2); }); + it('should generate interest_content_batch with a run-scoped uniqueKey', () => { + const actual = generateNotificationV2( + NotificationType.InterestContentBatch, + { + userIds: [userId], + interest: { id: 'i1', query: 'cool rust projects' }, + count: 3, + dedupKey: 'i1:1700000000000', + }, + ); + + expect(actual.notification.type).toEqual( + NotificationType.InterestContentBatch, + ); + expect(actual.notification.referenceId).toEqual('i1'); + expect(actual.notification.uniqueKey).toEqual('i1:1700000000000'); + }); + + it('should fall back to the interest id when interest_content_batch has no dedupKey', () => { + const actual = generateNotificationV2( + NotificationType.InterestContentBatch, + { + userIds: [userId], + interest: { id: 'i1', query: 'cool rust projects' }, + count: 3, + }, + ); + + expect(actual.notification.uniqueKey).toEqual('i1'); + }); + + it('should deliver one interest_content_batch per run and dedupe within a run', async () => { + const build = (dedupKey: string) => + generateNotificationV2(NotificationType.InterestContentBatch, { + userIds: [userId], + interest: { id: 'i1', query: 'cool rust projects' }, + count: 3, + dedupKey, + }); + + await con.transaction(async (manager) => { + await storeNotificationBundleV2(manager, build('i1:run-1'), 'i1:run-1'); + await storeNotificationBundleV2(manager, build('i1:run-1'), 'i1:run-1'); + await storeNotificationBundleV2(manager, build('i1:run-2'), 'i1:run-2'); + }); + + const notifications = await con + .getRepository(NotificationV2) + .findBy({ type: NotificationType.InterestContentBatch }); + expect(notifications.length).toEqual(2); + + const userNotifications = await con + .getRepository(UserNotification) + .findBy({ userId }); + expect(userNotifications.length).toEqual(2); + }); + it('should generate squad_new_comment notification', () => { const type = NotificationType.SquadNewComment; const ctx: NotificationCommenterContext = { diff --git a/src/common/interest/discoverExternalUrls.ts b/src/common/interest/discoverExternalUrls.ts new file mode 100644 index 0000000000..bf79d8b6b8 --- /dev/null +++ b/src/common/interest/discoverExternalUrls.ts @@ -0,0 +1,125 @@ +import type { FastifyBaseLogger } from 'fastify'; +import { anthropicClient } from '../../integrations/anthropic/client'; +import type { UserInterest } from '../../entity/UserInterest'; + +export type DiscoveredUrl = { + url: string; + title: string; + rationale: string; + score: number; +}; + +const DEFAULT_DISCOVERY_LIMIT = 5; +const WEB_SEARCH_TOOL_TYPE = + process.env.INTEREST_AGENT_WEB_SEARCH_TOOL || 'web_search_20250305'; + +const extractJsonArray = (text: string): string => { + const start = text.indexOf('['); + const end = text.lastIndexOf(']'); + return start !== -1 && end !== -1 ? text.slice(start, end + 1) : '[]'; +}; + +const isHttpUrl = (value: unknown): value is string => + typeof value === 'string' && /^https?:\/\//i.test(value); + +export const discoverExternalUrls = async ({ + interest, + query, + limit, + logger, +}: { + interest: Pick; + query: string; + limit?: number; + logger: FastifyBaseLogger; +}): Promise => { + const log = logger.child({ provider: 'interest agent' }); + const max = Math.max(1, Math.min(limit ?? DEFAULT_DISCOVERY_LIMIT, 10)); + + const webTool: Record = { + type: WEB_SEARCH_TOOL_TYPE, + name: 'web_search', + max_uses: max, + }; + + const system = [ + 'You are the daily.dev Interest Agent external scout.', + 'Use the web_search tool to find high-quality, recent content that genuinely matches the user interest.', + 'Prefer articles, docs, and notable repositories. Do not return daily.dev URLs.', + 'Be strict: only include results that are genuinely about the interest.', + `Return at most ${max} results.`, + 'Reply with ONLY a JSON array, no prose: ' + + '[{"url": string, "title": string, "rationale": short string, "score": number 0-1 for how well it matches the interest}].', + 'If nothing good is found, reply with [].', + ].join('\n'); + + let response; + try { + response = await anthropicClient.createMessage({ + model: process.env.INTEREST_AGENT_MODEL || 'claude-opus-4-8', + max_tokens: 4096, + system, + messages: [ + { + role: 'user', + content: `Interest: "${interest.query}".\nSearch query: "${query}".\nFind matching content now.`, + }, + ], + tools: [webTool], + }); + } catch (err) { + log.warn( + { interestId: interest.id, query, err }, + 'interest discovery failed', + ); + return []; + } + + const text = (response.content as Array<{ type?: string; text?: string }>) + .map((block) => (block?.type === 'text' ? (block.text ?? '') : '')) + .filter(Boolean) + .join('\n') + .trim(); + + let parsed: unknown; + try { + parsed = JSON.parse(extractJsonArray(text)); + } catch { + log.warn( + { interestId: interest.id, query }, + 'interest discovery parse failed', + ); + return []; + } + + if (!Array.isArray(parsed)) { + return []; + } + + const seen = new Set(); + const results: DiscoveredUrl[] = []; + for (const item of parsed as Array>) { + if (!isHttpUrl(item?.url) || seen.has(item.url)) { + continue; + } + seen.add(item.url); + results.push({ + url: item.url, + title: typeof item.title === 'string' ? item.title : '', + rationale: + typeof item.rationale === 'string' + ? item.rationale + : 'Discovered on the web for this interest', + score: typeof item.score === 'number' ? item.score : 0, + }); + if (results.length >= max) { + break; + } + } + + log.info( + { interestId: interest.id, query, discovered: results.length }, + 'interest discovery complete', + ); + return results; +}; diff --git a/src/common/interest/runInterestAgent.ts b/src/common/interest/runInterestAgent.ts index 36c8182be5..e3a69ee2c6 100644 --- a/src/common/interest/runInterestAgent.ts +++ b/src/common/interest/runInterestAgent.ts @@ -12,7 +12,7 @@ import { } from '@dailydotdev/schema'; import type { DataSource } from 'typeorm'; import type { FastifyBaseLogger } from 'fastify'; -import { In } from 'typeorm'; +import { In, MoreThanOrEqual } from 'typeorm'; import { mimirClient } from '../../integrations/mimir/clients'; import { mimirFilterBuilder } from '../../integrations/mimir/filters'; import { getBragiClient } from '../../integrations/bragi/clients'; @@ -23,10 +23,12 @@ import { FeedTag } from '../../entity/FeedTag'; import { InterestFinding, InterestFindingStatus, + InterestFindingOrigin, } from '../../entity/InterestFinding'; import { InterestFeedback } from '../../entity/InterestFeedback'; import type { UserInterest } from '../../entity/UserInterest'; -import { insertFreeformPost } from '../post'; +import { insertFreeformPost, getExistingPost } from '../post'; +import { createExternalLink } from '../../entity/posts/utils'; import { getDiscussionLink } from '../links'; import { markdown } from '../markdown'; import { updateFlagsStatement } from '../utils'; @@ -38,9 +40,13 @@ import { DEFAULT_INTEREST_MAX_TAGS, } from './feedTags'; import { createInterestAgentModel } from './agentModel'; +import { discoverExternalUrls } from './discoverExternalUrls'; +import { ONE_DAY_IN_SECONDS } from '../constants'; const DEFAULT_SEARCH_LIMIT = 10; const SEARCH_VERSION = 3; +const DEFAULT_MAX_WEB_SEARCHES_PER_RUN = 3; +const DEFAULT_MAX_DISCOVERIES_PER_DAY = 30; export type InterestAgentRunResult = { findingsAdded: number; @@ -50,10 +56,15 @@ export type InterestAgentRunResult = { export const getInterestAgentTools = ( outputModes?: UserInterest['outputModes'], + sources?: UserInterest['sources'], ): string[] => { + const feed = outputModes?.feed ?? true; const tools = ['set_interest_tags', 'search_daily_dev']; - if (outputModes?.feed ?? true) { + if (feed) { tools.push('score_finding', 'add_to_feed'); + if (sources?.web) { + tools.push('discover_external'); + } } if (outputModes?.post ?? true) { tools.push('write_post'); @@ -68,6 +79,7 @@ const buildSystemPrompt = ( maxTags: number, ): string => { const { feed = true, post = true } = interest.outputModes ?? {}; + const externalEnabled = feed && !!interest.sources?.web; const steps = [ `1. Call set_interest_tags with the full set of daily.dev tag slugs that represent this interest (max ${maxTags}); it replaces the current set, so include the ones worth keeping and drop the rest.`, '2. Call search_daily_dev with a focused query derived from the interest.', @@ -77,6 +89,11 @@ const buildSystemPrompt = ( `${steps.length + 1}. For each promising result, call score_finding — it returns an audienceFit quality signal (0-1) that reflects general daily.dev content quality, NOT topical relevance to this interest.`, `${steps.length + 2}. Judge topical relevance yourself from the title and summary. Call add_to_feed only for results that genuinely match the interest, passing a relevance score (0-1) for how well the post matches the interest, plus a short rationale.`, ); + if (externalEnabled) { + steps.push( + `${steps.length + 1}. If daily.dev doesn't have enough strong matches, call discover_external with a focused search query to pull in external web content — it is ingested into daily.dev and added to the feed automatically.`, + ); + } } if (post) { steps.push( @@ -87,7 +104,9 @@ const buildSystemPrompt = ( return [ 'You are the daily.dev Interest Agent. You hunt for content matching a single user interest, score it, and deliver it.', `The interest is: "${interest.query}".`, - 'Work only with daily.dev content in this run — do not invent URLs or reference external sources.', + externalEnabled + ? 'Prefer daily.dev content; use discover_external only to fill gaps. Never invent URLs — only use urls returned by search_daily_dev or discover_external.' + : 'Work only with daily.dev content in this run — do not invent URLs or reference external sources.', 'Be strict about topical relevance: a well-written post about a different topic must NOT be surfaced. Only add_to_feed posts that are genuinely about the interest.', `FOMO threshold is ${interest.fomoThreshold ?? 0.5} (0 = surface everything, 1 = only the very best). Only add_to_feed items whose relevance score is at or above this threshold.`, currentTags.length @@ -133,6 +152,7 @@ export const runInterestAgent = async ({ const scores = new Map(); const addedPostIds = new Set(); + let discoverCalls = 0; const registerTools = (pi: ExtensionAPI) => { pi.registerTool({ @@ -304,6 +324,7 @@ export const runInterestAgent = async ({ score, rationale: params.rationale, status: InterestFindingStatus.New, + origin: InterestFindingOrigin.Search, }) .orUpdate(['score', 'rationale', 'status'], ['interestId', 'postId']) .execute(); @@ -403,6 +424,109 @@ export const runInterestAgent = async ({ }; }, }); + + pi.registerTool({ + name: 'discover_external', + label: 'Discover external content', + description: + "Search the web for content matching the interest that is NOT already on daily.dev. Pass a focused search query. Matching pages are ingested into daily.dev and added to the interest's feed as findings. Use this to broaden beyond daily.dev's existing content.", + parameters: Type.Object({ + query: Type.String(), + limit: Type.Optional(Type.Number()), + }), + execute: async (_id, params) => { + const respond = (payload: Record) => ({ + content: [{ type: 'text', text: JSON.stringify(payload) }], + details: {}, + }); + + const maxCalls = + remoteConfig.vars.interestAgentMaxWebSearchesPerRun ?? + DEFAULT_MAX_WEB_SEARCHES_PER_RUN; + discoverCalls += 1; + if (discoverCalls > maxCalls) { + return respond({ error: 'web_search_budget_exhausted', maxCalls }); + } + + const maxPerDay = + remoteConfig.vars.interestAgentMaxDiscoveriesPerDay ?? + DEFAULT_MAX_DISCOVERIES_PER_DAY; + const since = new Date(Date.now() - ONE_DAY_IN_SECONDS * 1000); + const discoveredToday = await con.getRepository(InterestFinding).count({ + where: { + interestId: interest.id, + origin: InterestFindingOrigin.Discovery, + createdAt: MoreThanOrEqual(since), + }, + }); + const remaining = maxPerDay - discoveredToday; + if (remaining <= 0) { + return respond({ error: 'daily_discovery_cap_reached', maxPerDay }); + } + + const candidates = await discoverExternalUrls({ + interest, + query: params.query, + limit: Math.min(params.limit ?? remaining, remaining), + logger, + }); + const threshold = interest.fomoThreshold ?? 0.5; + let added = 0; + for (const candidate of candidates) { + if (added >= remaining) { + break; + } + if (candidate.score < threshold) { + continue; + } + const existing = await getExistingPost(con, { + url: candidate.url, + canonicalUrl: candidate.url, + }); + if (existing?.deleted) { + continue; + } + let postId = existing?.id; + if (!postId) { + postId = await generateShortId(); + await createExternalLink({ + con, + args: { + id: postId, + title: candidate.title || undefined, + url: candidate.url, + canonicalUrl: candidate.url, + authorId: interest.userId, + originalUrl: candidate.url, + }, + }); + } + await con + .getRepository(InterestFinding) + .createQueryBuilder() + .insert() + .values({ + id: await generateShortId(), + interestId: interest.id, + postId, + score: candidate.score, + rationale: candidate.rationale, + status: InterestFindingStatus.New, + origin: InterestFindingOrigin.Discovery, + }) + .orIgnore() + .execute(); + addedPostIds.add(postId); + state.findingsAdded += 1; + added += 1; + } + log.info( + { interestId: interest.id, query: params.query, added }, + 'interest agent discover_external', + ); + return respond({ discovered: candidates.length, added }); + }, + }); }; const feedbackRows = await con.getRepository(InterestFeedback).find({ @@ -423,7 +547,10 @@ export const runInterestAgent = async ({ : []; const currentTags = currentTagRows.map((row) => row.tag); - const activeTools = getInterestAgentTools(interest.outputModes); + const activeTools = getInterestAgentTools( + interest.outputModes, + interest.sources, + ); const resourceLoader = new DefaultResourceLoader({ cwd: agentDir, diff --git a/src/entity/InterestFinding.ts b/src/entity/InterestFinding.ts index 66de7ba5a5..ad93dd733d 100644 --- a/src/entity/InterestFinding.ts +++ b/src/entity/InterestFinding.ts @@ -17,6 +17,12 @@ export enum InterestFindingStatus { Dismissed = 'dismissed', } +export enum InterestFindingOrigin { + Search = 'search', + Live = 'live', + Discovery = 'discovery', +} + @Entity() @Index('IDX_interest_finding_interest_id_post_id', ['interestId', 'postId'], { unique: true, @@ -41,6 +47,9 @@ export class InterestFinding { @Column({ type: 'text', default: InterestFindingStatus.New }) status: InterestFindingStatus; + @Column({ type: 'text', default: InterestFindingOrigin.Search }) + origin: InterestFindingOrigin; + @CreateDateColumn() createdAt: Date; diff --git a/src/migration/1784719854693-AddInterestFindingOrigin.ts b/src/migration/1784719854693-AddInterestFindingOrigin.ts new file mode 100644 index 0000000000..5b45f6670e --- /dev/null +++ b/src/migration/1784719854693-AddInterestFindingOrigin.ts @@ -0,0 +1,17 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddInterestFindingOrigin1784719854693 implements MigrationInterface { + name = 'AddInterestFindingOrigin1784719854693'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "interest_finding" ADD "origin" text NOT NULL DEFAULT 'search'`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "interest_finding" DROP COLUMN "origin"`, + ); + } +} diff --git a/src/notifications/generate.ts b/src/notifications/generate.ts index f750735fa6..03dc77f519 100644 --- a/src/notifications/generate.ts +++ b/src/notifications/generate.ts @@ -649,7 +649,7 @@ export const generateNotificationMap: Record< .icon(NotificationIcon.Bell) .referenceInterest(ctx.interest) .targetUrl(`${process.env.COMMENTS_PREFIX}/agent/${ctx.interest.id}`) - .uniqueKey(ctx.interest.id); + .uniqueKey(ctx.dedupKey ?? ctx.interest.id); }, user_follow: (builder, ctx: NotificationUserContext) => { const userName = ctx.user.name || ctx.user.username; diff --git a/src/remoteConfig.ts b/src/remoteConfig.ts index ad13aac0a0..39fb3f3049 100644 --- a/src/remoteConfig.ts +++ b/src/remoteConfig.ts @@ -56,6 +56,8 @@ export type RemoteConfigValue = { dailyBriefLimit: number; interestAgentMaxInterestsPerPost: number; interestAgentMaxTags: number; + interestAgentMaxWebSearchesPerRun: number; + interestAgentMaxDiscoveriesPerDay: number; dailyFeedCacheKey: string; superAgentTrial: SuperAgentTrialConfig; digestPostEnabled: boolean; diff --git a/src/workers/interest/postVisibleInterestMatch.ts b/src/workers/interest/postVisibleInterestMatch.ts index a4cdd43e74..25b95f7a13 100644 --- a/src/workers/interest/postVisibleInterestMatch.ts +++ b/src/workers/interest/postVisibleInterestMatch.ts @@ -4,6 +4,7 @@ import { UserInterest, UserInterestStatus } from '../../entity/UserInterest'; import { InterestFinding, InterestFindingStatus, + InterestFindingOrigin, } from '../../entity/InterestFinding'; import { PostKeyword } from '../../entity/PostKeyword'; import { KeywordStatus } from '../../entity/Keyword'; @@ -143,6 +144,7 @@ export const postVisibleInterestMatchWorker: TypedWorker<'api.v1.post-visible'> score: relevance.score, rationale: relevance.rationale ?? 'Matched a newly published post', status: InterestFindingStatus.New, + origin: InterestFindingOrigin.Live, }) .orIgnore() .execute(); From f7c9608c6805ea6aac4289d35bb01da9f29d52f9 Mon Sep 17 00:00:00 2001 From: capJavert Date: Wed, 22 Jul 2026 15:45:12 +0200 Subject: [PATCH 2/5] feat: review feedback --- .../common/interest/discoverExternalUrls.ts | 31 +++++ src/common/interest/discoverExternalUrls.ts | 18 ++- src/common/interest/runInterestAgent.ts | 122 +++++++++++------- .../1784719854693-AddInterestFindingOrigin.ts | 4 +- 4 files changed, 124 insertions(+), 51 deletions(-) diff --git a/__tests__/common/interest/discoverExternalUrls.ts b/__tests__/common/interest/discoverExternalUrls.ts index 83cd7dc3d9..bf2992a0d7 100644 --- a/__tests__/common/interest/discoverExternalUrls.ts +++ b/__tests__/common/interest/discoverExternalUrls.ts @@ -52,6 +52,37 @@ describe('discoverExternalUrls', () => { expect(result.map((r) => r.url)).toEqual(['https://a.com']); }); + it('drops daily.dev urls so the agent cannot ingest our own content', async () => { + mockContent( + JSON.stringify([ + { + url: 'https://daily.dev/posts/x', + title: 'own', + rationale: 'r', + score: 0.9, + }, + { + url: 'https://app.daily.dev/posts/y', + title: 'own2', + rationale: 'r', + score: 0.9, + }, + { + url: 'https://external.com/a', + title: 'ext', + rationale: 'r', + score: 0.9, + }, + ]), + ); + const result = await discoverExternalUrls({ + interest, + query: 'zig', + logger, + }); + expect(result.map((r) => r.url)).toEqual(['https://external.com/a']); + }); + it('returns an empty array when the reply is not a JSON array', async () => { mockContent('I could not find anything relevant.'); const result = await discoverExternalUrls({ diff --git a/src/common/interest/discoverExternalUrls.ts b/src/common/interest/discoverExternalUrls.ts index bf79d8b6b8..f70b6f070b 100644 --- a/src/common/interest/discoverExternalUrls.ts +++ b/src/common/interest/discoverExternalUrls.ts @@ -1,5 +1,6 @@ import type { FastifyBaseLogger } from 'fastify'; import { anthropicClient } from '../../integrations/anthropic/client'; +import type { AnthropicResponse } from '../../integrations/anthropic/types'; import type { UserInterest } from '../../entity/UserInterest'; export type DiscoveredUrl = { @@ -22,6 +23,15 @@ const extractJsonArray = (text: string): string => { const isHttpUrl = (value: unknown): value is string => typeof value === 'string' && /^https?:\/\//i.test(value); +const isDailyDevUrl = (value: string): boolean => { + try { + const host = new URL(value).hostname.toLowerCase(); + return host === 'daily.dev' || host.endsWith('.daily.dev'); + } catch { + return false; + } +}; + export const discoverExternalUrls = async ({ interest, query, @@ -53,7 +63,7 @@ export const discoverExternalUrls = async ({ 'If nothing good is found, reply with [].', ].join('\n'); - let response; + let response: AnthropicResponse; try { response = await anthropicClient.createMessage({ model: process.env.INTEREST_AGENT_MODEL || 'claude-opus-4-8', @@ -99,7 +109,11 @@ export const discoverExternalUrls = async ({ const seen = new Set(); const results: DiscoveredUrl[] = []; for (const item of parsed as Array>) { - if (!isHttpUrl(item?.url) || seen.has(item.url)) { + if ( + !isHttpUrl(item?.url) || + seen.has(item.url) || + isDailyDevUrl(item.url) + ) { continue; } seen.add(item.url); diff --git a/src/common/interest/runInterestAgent.ts b/src/common/interest/runInterestAgent.ts index e3a69ee2c6..e11fcfdfd8 100644 --- a/src/common/interest/runInterestAgent.ts +++ b/src/common/interest/runInterestAgent.ts @@ -29,7 +29,8 @@ import { InterestFeedback } from '../../entity/InterestFeedback'; import type { UserInterest } from '../../entity/UserInterest'; import { insertFreeformPost, getExistingPost } from '../post'; import { createExternalLink } from '../../entity/posts/utils'; -import { getDiscussionLink } from '../links'; +import { getDiscussionLink, standardizeURL } from '../links'; +import { blockingBatchRunner } from '../async'; import { markdown } from '../markdown'; import { updateFlagsStatement } from '../utils'; import { generateShortId } from '../../ids'; @@ -47,6 +48,7 @@ const DEFAULT_SEARCH_LIMIT = 10; const SEARCH_VERSION = 3; const DEFAULT_MAX_WEB_SEARCHES_PER_RUN = 3; const DEFAULT_MAX_DISCOVERIES_PER_DAY = 30; +const DISCOVERY_BATCH_SIZE = 10; export type InterestAgentRunResult = { findingsAdded: number; @@ -471,55 +473,81 @@ export const runInterestAgent = async ({ logger, }); const threshold = interest.fomoThreshold ?? 0.5; - let added = 0; - for (const candidate of candidates) { - if (added >= remaining) { - break; - } + const seenCanonical = new Set(); + const eligible = candidates.reduce< + { + candidate: (typeof candidates)[number]; + url: string; + canonicalUrl: string; + }[] + >((acc, candidate) => { if (candidate.score < threshold) { - continue; + return acc; } - const existing = await getExistingPost(con, { - url: candidate.url, - canonicalUrl: candidate.url, - }); - if (existing?.deleted) { - continue; + const { url, canonicalUrl } = standardizeURL(candidate.url); + if (seenCanonical.has(canonicalUrl)) { + return acc; } - let postId = existing?.id; - if (!postId) { - postId = await generateShortId(); - await createExternalLink({ - con, - args: { - id: postId, - title: candidate.title || undefined, - url: candidate.url, - canonicalUrl: candidate.url, - authorId: interest.userId, - originalUrl: candidate.url, - }, - }); - } - await con - .getRepository(InterestFinding) - .createQueryBuilder() - .insert() - .values({ - id: await generateShortId(), - interestId: interest.id, - postId, - score: candidate.score, - rationale: candidate.rationale, - status: InterestFindingStatus.New, - origin: InterestFindingOrigin.Discovery, - }) - .orIgnore() - .execute(); - addedPostIds.add(postId); - state.findingsAdded += 1; - added += 1; - } + seenCanonical.add(canonicalUrl); + acc.push({ candidate, url, canonicalUrl }); + return acc; + }, []); + + let added = 0; + await blockingBatchRunner({ + data: eligible, + batchLimit: DISCOVERY_BATCH_SIZE, + runner: async (batch) => { + const inserted = await Promise.all( + batch.map(async ({ candidate, url, canonicalUrl }) => { + const existing = await getExistingPost(con, { + url, + canonicalUrl, + }); + if (existing?.deleted) { + return false; + } + let postId = existing?.id; + if (!postId) { + postId = await generateShortId(); + await createExternalLink({ + con, + args: { + id: postId, + title: candidate.title || undefined, + url, + canonicalUrl, + authorId: interest.userId, + originalUrl: candidate.url, + }, + }); + } + const insertResult = await con + .getRepository(InterestFinding) + .createQueryBuilder() + .insert() + .values({ + id: await generateShortId(), + interestId: interest.id, + postId, + score: candidate.score, + rationale: candidate.rationale, + status: InterestFindingStatus.New, + origin: InterestFindingOrigin.Discovery, + }) + .orIgnore() + .execute(); + if (!(insertResult.raw as unknown[])?.length) { + return false; + } + addedPostIds.add(postId); + state.findingsAdded += 1; + return true; + }), + ); + added += inserted.filter(Boolean).length; + }, + }); log.info( { interestId: interest.id, query: params.query, added }, 'interest agent discover_external', diff --git a/src/migration/1784719854693-AddInterestFindingOrigin.ts b/src/migration/1784719854693-AddInterestFindingOrigin.ts index 5b45f6670e..0f38ce4eac 100644 --- a/src/migration/1784719854693-AddInterestFindingOrigin.ts +++ b/src/migration/1784719854693-AddInterestFindingOrigin.ts @@ -5,13 +5,13 @@ export class AddInterestFindingOrigin1784719854693 implements MigrationInterface public async up(queryRunner: QueryRunner): Promise { await queryRunner.query( - `ALTER TABLE "interest_finding" ADD "origin" text NOT NULL DEFAULT 'search'`, + `ALTER TABLE "interest_finding" ADD COLUMN IF NOT EXISTS "origin" text NOT NULL DEFAULT 'search'`, ); } public async down(queryRunner: QueryRunner): Promise { await queryRunner.query( - `ALTER TABLE "interest_finding" DROP COLUMN "origin"`, + `ALTER TABLE "interest_finding" DROP COLUMN IF EXISTS "origin"`, ); } } From a63b3eb4b17a1fddec78074caf0009e94dd9f502 Mon Sep 17 00:00:00 2001 From: capJavert Date: Wed, 22 Jul 2026 17:00:48 +0200 Subject: [PATCH 3/5] feat: adjustments to content --- .../interest/discoverAndIngestExternal.ts | 180 ++++++++++++ __tests__/interests.ts | 5 +- src/common/interest/runInterestAgent.ts | 257 +++++++++++------- src/schema/interests.ts | 4 + 4 files changed, 342 insertions(+), 104 deletions(-) create mode 100644 __tests__/common/interest/discoverAndIngestExternal.ts diff --git a/__tests__/common/interest/discoverAndIngestExternal.ts b/__tests__/common/interest/discoverAndIngestExternal.ts new file mode 100644 index 0000000000..e92f5ac11d --- /dev/null +++ b/__tests__/common/interest/discoverAndIngestExternal.ts @@ -0,0 +1,180 @@ +import { DataSource } from 'typeorm'; +import type { FastifyBaseLogger } from 'fastify'; +import createOrGetConnection from '../../../src/db'; +import { saveFixtures } from '../../helpers'; +import { ArticlePost, Source, User } from '../../../src/entity'; +import { AgentSource } from '../../../src/entity/Source'; +import { SharePost } from '../../../src/entity/posts/SharePost'; +import { + UserInterest, + UserInterestStatus, +} from '../../../src/entity/UserInterest'; +import { + InterestFinding, + InterestFindingOrigin, + InterestFindingStatus, +} from '../../../src/entity/InterestFinding'; +import { usersFixture } from '../../fixture/user'; +import { postsFixture } from '../../fixture/post'; +import { sourcesFixture } from '../../fixture'; +import { remoteConfig } from '../../../src/remoteConfig'; +import { discoverExternalUrls } from '../../../src/common/interest/discoverExternalUrls'; +import { discoverAndIngestExternal } from '../../../src/common/interest/runInterestAgent'; + +jest.mock('../../../src/common/interest/discoverExternalUrls', () => ({ + discoverExternalUrls: jest.fn(), +})); + +let con: DataSource; + +const logger = { + child: () => ({ info: jest.fn(), warn: jest.fn() }), +} as unknown as FastifyBaseLogger; + +const interest = { + id: 'uir-d', + query: 'cool zig projects', + userId: '1', + sourceId: 'asrc-d', + fomoThreshold: 0.5, +}; + +const setCandidates = (candidates: unknown[]) => + (discoverExternalUrls as jest.Mock).mockResolvedValue(candidates); + +beforeAll(async () => { + con = await createOrGetConnection(); +}); + +beforeEach(async () => { + jest.restoreAllMocks(); + jest.clearAllMocks(); + await saveFixtures(con, User, usersFixture); + await saveFixtures(con, Source, sourcesFixture); + await saveFixtures(con, ArticlePost, postsFixture); + await con.getRepository(AgentSource).save({ + id: 'asrc-d', + name: 'agent source', + handle: 'agent-asrc-d', + private: true, + }); + await con.getRepository(UserInterest).save({ + id: 'uir-d', + userId: '1', + query: 'cool zig projects', + status: UserInterestStatus.Active, + sourceId: 'asrc-d', + fomoThreshold: 0.5, + }); +}); + +afterEach(() => { + remoteConfig.vars.interestAgentMaxDiscoveriesPerDay = undefined; +}); + +describe('discoverAndIngestExternal', () => { + it('creates an article, a share in the agent source, and a discovery finding pointing at the share', async () => { + setCandidates([ + { url: 'https://ext.com/a', title: 'A', rationale: 'why', score: 0.9 }, + ]); + + const result = await discoverAndIngestExternal({ + con, + logger, + interest, + query: 'zig', + }); + + expect(result.added).toBe(1); + + const findings = await con + .getRepository(InterestFinding) + .findBy({ interestId: 'uir-d' }); + expect(findings).toHaveLength(1); + expect(findings[0].origin).toBe(InterestFindingOrigin.Discovery); + + const share = await con + .getRepository(SharePost) + .findOneByOrFail({ id: findings[0].postId }); + expect(share.sourceId).toBe('asrc-d'); + expect(share.private).toBe(true); + expect(share.visible).toBe(true); + + const article = await con + .getRepository(ArticlePost) + .findOneByOrFail({ id: share.sharedPostId as string }); + expect(article.url).toBe('https://ext.com/a'); + }); + + it('does not inflate the count when the same url is rediscovered (dedup, no duplicate share/finding)', async () => { + setCandidates([ + { url: 'https://ext.com/a', title: 'A', rationale: 'why', score: 0.9 }, + ]); + + const first = await discoverAndIngestExternal({ + con, + logger, + interest, + query: 'zig', + }); + const second = await discoverAndIngestExternal({ + con, + logger, + interest, + query: 'zig', + }); + + expect(first.added).toBe(1); + expect(second.added).toBe(0); + expect( + await con.getRepository(InterestFinding).countBy({ interestId: 'uir-d' }), + ).toBe(1); + expect( + await con.getRepository(SharePost).countBy({ sourceId: 'asrc-d' }), + ).toBe(1); + }); + + it('skips candidates below the fomo threshold', async () => { + setCandidates([ + { url: 'https://ext.com/b', title: 'B', rationale: 'why', score: 0.2 }, + ]); + + const result = await discoverAndIngestExternal({ + con, + logger, + interest, + query: 'zig', + }); + + expect(result.added).toBe(0); + expect( + await con.getRepository(InterestFinding).countBy({ interestId: 'uir-d' }), + ).toBe(0); + }); + + it('short-circuits when the daily discovery cap is reached and does not search', async () => { + remoteConfig.vars.interestAgentMaxDiscoveriesPerDay = 1; + + await con.getRepository(InterestFinding).save({ + id: 'finding-cap', + interestId: 'uir-d', + postId: postsFixture[0].id, + score: 0.9, + status: InterestFindingStatus.New, + origin: InterestFindingOrigin.Discovery, + }); + setCandidates([ + { url: 'https://ext.com/c', title: 'C', rationale: 'why', score: 0.9 }, + ]); + + const result = await discoverAndIngestExternal({ + con, + logger, + interest, + query: 'zig', + }); + + expect(result.added).toBe(0); + expect(discoverExternalUrls).not.toHaveBeenCalled(); + }); +}); diff --git a/__tests__/interests.ts b/__tests__/interests.ts index a7c3ead91f..04846741fc 100644 --- a/__tests__/interests.ts +++ b/__tests__/interests.ts @@ -11,6 +11,7 @@ import { testQueryErrorCode, } from './helpers'; import { ArticlePost, Source, User } from '../src/entity'; +import { FreeformPost } from '../src/entity/posts/FreeformPost'; import { Feed, FeedOrigin } from '../src/entity/Feed'; import { AgentSource, SourceType, SourceUser } from '../src/entity/Source'; import { UserInterest, UserInterestStatus } from '../src/entity/UserInterest'; @@ -540,12 +541,12 @@ describe('query interestPosts', () => { status: UserInterestStatus.Active, sourceId: 'isrc-1', }); - await saveFixtures(con, ArticlePost, [ + await saveFixtures(con, FreeformPost, [ { id: 'ipost-1', shortId: 'ipost-1', title: 'Interest summary', - url: 'http://interest.com/1', + content: 'Interest summary content', sourceId: 'isrc-1', private: true, showOnFeed: false, diff --git a/src/common/interest/runInterestAgent.ts b/src/common/interest/runInterestAgent.ts index e11fcfdfd8..69c9b106b5 100644 --- a/src/common/interest/runInterestAgent.ts +++ b/src/common/interest/runInterestAgent.ts @@ -28,7 +28,8 @@ import { import { InterestFeedback } from '../../entity/InterestFeedback'; import type { UserInterest } from '../../entity/UserInterest'; import { insertFreeformPost, getExistingPost } from '../post'; -import { createExternalLink } from '../../entity/posts/utils'; +import { createExternalLink, createSharePost } from '../../entity/posts/utils'; +import { SharePost } from '../../entity/posts/SharePost'; import { getDiscussionLink, standardizeURL } from '../links'; import { blockingBatchRunner } from '../async'; import { markdown } from '../markdown'; @@ -41,7 +42,10 @@ import { DEFAULT_INTEREST_MAX_TAGS, } from './feedTags'; import { createInterestAgentModel } from './agentModel'; -import { discoverExternalUrls } from './discoverExternalUrls'; +import { + discoverExternalUrls, + type DiscoveredUrl, +} from './discoverExternalUrls'; import { ONE_DAY_IN_SECONDS } from '../constants'; const DEFAULT_SEARCH_LIMIT = 10; @@ -128,6 +132,148 @@ const buildSystemPrompt = ( .join('\n'); }; +export const discoverAndIngestExternal = async ({ + con, + logger, + interest, + query, + limit, +}: { + con: DataSource; + logger: FastifyBaseLogger; + interest: Pick< + UserInterest, + 'id' | 'query' | 'userId' | 'sourceId' | 'fomoThreshold' + >; + query: string; + limit?: number; +}): Promise<{ discovered: number; added: number; postIds: string[] }> => { + if (!interest.sourceId) { + return { discovered: 0, added: 0, postIds: [] }; + } + const sourceId = interest.sourceId; + + const maxPerDay = + remoteConfig.vars.interestAgentMaxDiscoveriesPerDay ?? + DEFAULT_MAX_DISCOVERIES_PER_DAY; + const since = new Date(Date.now() - ONE_DAY_IN_SECONDS * 1000); + const discoveredToday = await con.getRepository(InterestFinding).count({ + where: { + interestId: interest.id, + origin: InterestFindingOrigin.Discovery, + createdAt: MoreThanOrEqual(since), + }, + }); + const remaining = maxPerDay - discoveredToday; + if (remaining <= 0) { + return { discovered: 0, added: 0, postIds: [] }; + } + + const candidates = await discoverExternalUrls({ + interest, + query, + limit: Math.min(limit ?? remaining, remaining), + logger, + }); + + const threshold = interest.fomoThreshold ?? 0.5; + const seenCanonical = new Set(); + const eligible = candidates.reduce< + { candidate: DiscoveredUrl; url: string; canonicalUrl: string }[] + >((acc, candidate) => { + if (candidate.score < threshold) { + return acc; + } + const { url, canonicalUrl } = standardizeURL(candidate.url); + if (seenCanonical.has(canonicalUrl)) { + return acc; + } + seenCanonical.add(canonicalUrl); + acc.push({ candidate, url, canonicalUrl }); + return acc; + }, []); + + const postIds: string[] = []; + await blockingBatchRunner({ + data: eligible, + batchLimit: DISCOVERY_BATCH_SIZE, + runner: async (batch) => { + const results = await Promise.all( + batch.map(async ({ candidate, url, canonicalUrl }) => { + const existing = await getExistingPost(con, { url, canonicalUrl }); + if (existing?.deleted) { + return null; + } + let articleId = existing?.id; + if (!articleId) { + articleId = await generateShortId(); + await createExternalLink({ + con, + args: { + id: articleId, + title: candidate.title || undefined, + url, + canonicalUrl, + authorId: interest.userId, + originalUrl: candidate.url, + }, + }); + } + + const existingShare = await con.getRepository(SharePost).findOne({ + select: ['id'], + where: { sourceId, sharedPostId: articleId, deleted: false }, + }); + const shareId = + existingShare?.id ?? + ( + await createSharePost({ + con, + args: { + authorId: interest.userId, + sourceId, + postId: articleId, + visible: true, + }, + }) + ).id; + + const insertResult = await con + .getRepository(InterestFinding) + .createQueryBuilder() + .insert() + .values({ + id: await generateShortId(), + interestId: interest.id, + postId: shareId, + score: candidate.score, + rationale: candidate.rationale, + status: InterestFindingStatus.New, + origin: InterestFindingOrigin.Discovery, + }) + .orIgnore() + .execute(); + return (insertResult.raw as unknown[])?.length ? shareId : null; + }), + ); + for (const shareId of results) { + if (shareId) { + postIds.push(shareId); + } + } + }, + }); + + logger + .child({ provider: 'interest agent' }) + .info( + { interestId: interest.id, query, added: postIds.length }, + 'interest agent discover_external', + ); + + return { discovered: candidates.length, added: postIds.length, postIds }; +}; + export const runInterestAgent = async ({ con, logger, @@ -450,109 +596,16 @@ export const runInterestAgent = async ({ return respond({ error: 'web_search_budget_exhausted', maxCalls }); } - const maxPerDay = - remoteConfig.vars.interestAgentMaxDiscoveriesPerDay ?? - DEFAULT_MAX_DISCOVERIES_PER_DAY; - const since = new Date(Date.now() - ONE_DAY_IN_SECONDS * 1000); - const discoveredToday = await con.getRepository(InterestFinding).count({ - where: { - interestId: interest.id, - origin: InterestFindingOrigin.Discovery, - createdAt: MoreThanOrEqual(since), - }, - }); - const remaining = maxPerDay - discoveredToday; - if (remaining <= 0) { - return respond({ error: 'daily_discovery_cap_reached', maxPerDay }); - } - - const candidates = await discoverExternalUrls({ + const result = await discoverAndIngestExternal({ + con, + logger, interest, query: params.query, - limit: Math.min(params.limit ?? remaining, remaining), - logger, - }); - const threshold = interest.fomoThreshold ?? 0.5; - const seenCanonical = new Set(); - const eligible = candidates.reduce< - { - candidate: (typeof candidates)[number]; - url: string; - canonicalUrl: string; - }[] - >((acc, candidate) => { - if (candidate.score < threshold) { - return acc; - } - const { url, canonicalUrl } = standardizeURL(candidate.url); - if (seenCanonical.has(canonicalUrl)) { - return acc; - } - seenCanonical.add(canonicalUrl); - acc.push({ candidate, url, canonicalUrl }); - return acc; - }, []); - - let added = 0; - await blockingBatchRunner({ - data: eligible, - batchLimit: DISCOVERY_BATCH_SIZE, - runner: async (batch) => { - const inserted = await Promise.all( - batch.map(async ({ candidate, url, canonicalUrl }) => { - const existing = await getExistingPost(con, { - url, - canonicalUrl, - }); - if (existing?.deleted) { - return false; - } - let postId = existing?.id; - if (!postId) { - postId = await generateShortId(); - await createExternalLink({ - con, - args: { - id: postId, - title: candidate.title || undefined, - url, - canonicalUrl, - authorId: interest.userId, - originalUrl: candidate.url, - }, - }); - } - const insertResult = await con - .getRepository(InterestFinding) - .createQueryBuilder() - .insert() - .values({ - id: await generateShortId(), - interestId: interest.id, - postId, - score: candidate.score, - rationale: candidate.rationale, - status: InterestFindingStatus.New, - origin: InterestFindingOrigin.Discovery, - }) - .orIgnore() - .execute(); - if (!(insertResult.raw as unknown[])?.length) { - return false; - } - addedPostIds.add(postId); - state.findingsAdded += 1; - return true; - }), - ); - added += inserted.filter(Boolean).length; - }, + limit: params.limit, }); - log.info( - { interestId: interest.id, query: params.query, added }, - 'interest agent discover_external', - ); - return respond({ discovered: candidates.length, added }); + result.postIds.forEach((postId) => addedPostIds.add(postId)); + state.findingsAdded += result.added; + return respond({ discovered: result.discovered, added: result.added }); }, }); }; diff --git a/src/schema/interests.ts b/src/schema/interests.ts index d266791ae9..84bd7f61e2 100644 --- a/src/schema/interests.ts +++ b/src/schema/interests.ts @@ -19,6 +19,7 @@ import { triggerTypedEvent } from '../common/typedPubsub'; import { queryReadReplica } from '../common/queryReadReplica'; import { GQLEmptyResponse } from './common'; import type { GQLPost } from './posts'; +import { PostType } from '../entity/posts/Post'; import { createInterestSchema, interestIdSchema, @@ -266,6 +267,9 @@ export const resolvers: IResolvers = { .where(`${builder.alias}."sourceId" = :sourceId`, { sourceId: interest.sourceId, }) + .andWhere(`${builder.alias}.type = :type`, { + type: PostType.Freeform, + }) .andWhere(`${builder.alias}.deleted = false`) .orderBy(`${builder.alias}."createdAt"`, 'DESC'); return builder; From 40b957519b7716efc2c35ba262ccd2da9b512cc7 Mon Sep 17 00:00:00 2001 From: capJavert Date: Wed, 22 Jul 2026 20:30:48 +0200 Subject: [PATCH 4/5] feat: make web search on by default --- __tests__/interests.ts | 2 +- src/entity/UserInterest.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/__tests__/interests.ts b/__tests__/interests.ts index 04846741fc..6a6ff8da3c 100644 --- a/__tests__/interests.ts +++ b/__tests__/interests.ts @@ -376,7 +376,7 @@ describe('mutation updateInterest', () => { query: 'cool zig projects', status: UserInterestStatus.Active, fomoThreshold: 0.5, - sources: { dailyDev: true, web: false, github: false }, + sources: { dailyDev: true, web: true, github: false }, outputModes: { feed: true, post: true, diff --git a/src/entity/UserInterest.ts b/src/entity/UserInterest.ts index ab612c9d13..d6aac1fd8d 100644 --- a/src/entity/UserInterest.ts +++ b/src/entity/UserInterest.ts @@ -34,7 +34,7 @@ export type UserInterestOutputModes = { export const defaultUserInterestSources: UserInterestSources = { dailyDev: true, - web: false, + web: true, github: false, }; From a1c4462a8b85fc74ac51c60adc2c4f005419c545 Mon Sep 17 00:00:00 2001 From: capJavert Date: Thu, 23 Jul 2026 08:24:31 +0200 Subject: [PATCH 5/5] feat: agent default state and guards --- .../interest/discoverAndIngestExternal.ts | 23 +++++++++++++++++++ src/common/interest/runInterestAgent.ts | 4 ++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/__tests__/common/interest/discoverAndIngestExternal.ts b/__tests__/common/interest/discoverAndIngestExternal.ts index e92f5ac11d..869d687fa6 100644 --- a/__tests__/common/interest/discoverAndIngestExternal.ts +++ b/__tests__/common/interest/discoverAndIngestExternal.ts @@ -37,6 +37,7 @@ const interest = { userId: '1', sourceId: 'asrc-d', fomoThreshold: 0.5, + sources: { dailyDev: true, web: true, github: false }, }; const setCandidates = (candidates: unknown[]) => @@ -134,6 +135,28 @@ describe('discoverAndIngestExternal', () => { ).toBe(1); }); + it('does nothing and does not search when the web source is off', async () => { + setCandidates([ + { url: 'https://ext.com/x', title: 'X', rationale: 'why', score: 0.9 }, + ]); + + const result = await discoverAndIngestExternal({ + con, + logger, + interest: { + ...interest, + sources: { dailyDev: true, web: false, github: false }, + }, + query: 'zig', + }); + + expect(result.added).toBe(0); + expect(discoverExternalUrls).not.toHaveBeenCalled(); + expect( + await con.getRepository(InterestFinding).countBy({ interestId: 'uir-d' }), + ).toBe(0); + }); + it('skips candidates below the fomo threshold', async () => { setCandidates([ { url: 'https://ext.com/b', title: 'B', rationale: 'why', score: 0.2 }, diff --git a/src/common/interest/runInterestAgent.ts b/src/common/interest/runInterestAgent.ts index 69c9b106b5..8d8fccaddb 100644 --- a/src/common/interest/runInterestAgent.ts +++ b/src/common/interest/runInterestAgent.ts @@ -143,12 +143,12 @@ export const discoverAndIngestExternal = async ({ logger: FastifyBaseLogger; interest: Pick< UserInterest, - 'id' | 'query' | 'userId' | 'sourceId' | 'fomoThreshold' + 'id' | 'query' | 'userId' | 'sourceId' | 'fomoThreshold' | 'sources' >; query: string; limit?: number; }): Promise<{ discovered: number; added: number; postIds: string[] }> => { - if (!interest.sourceId) { + if (!interest.sources?.web || !interest.sourceId) { return { discovered: 0, added: 0, postIds: [] }; } const sourceId = interest.sourceId;