feat: web search and notification adjustment

__tests__/common/interest/discoverAndIngestExternal.ts

@@ -0,0 +1,203 @@
+import { DataSource } from 'typeorm';
+import type { FastifyBaseLogger } from 'fastify';
+import createOrGetConnection from '../../../src/db';
+import { saveFixtures } from '../../helpers';
+import { ArticlePost, Source, User } from '../../../src/entity';
+import { AgentSource } from '../../../src/entity/Source';
+import { SharePost } from '../../../src/entity/posts/SharePost';
+import {
+  UserInterest,
+  UserInterestStatus,
+} from '../../../src/entity/UserInterest';
+import {
+  InterestFinding,
+  InterestFindingOrigin,
+  InterestFindingStatus,
+} from '../../../src/entity/InterestFinding';
+import { usersFixture } from '../../fixture/user';
+import { postsFixture } from '../../fixture/post';
+import { sourcesFixture } from '../../fixture';
+import { remoteConfig } from '../../../src/remoteConfig';
+import { discoverExternalUrls } from '../../../src/common/interest/discoverExternalUrls';
+import { discoverAndIngestExternal } from '../../../src/common/interest/runInterestAgent';
+
+jest.mock('../../../src/common/interest/discoverExternalUrls', () => ({
+  discoverExternalUrls: jest.fn(),
+}));
+
+let con: DataSource;
+
+const logger = {
+  child: () => ({ info: jest.fn(), warn: jest.fn() }),
+} as unknown as FastifyBaseLogger;
+
+const interest = {
+  id: 'uir-d',
+  query: 'cool zig projects',
+  userId: '1',
+  sourceId: 'asrc-d',
+  fomoThreshold: 0.5,
+  sources: { dailyDev: true, web: true, github: false },
+};
+
+const setCandidates = (candidates: unknown[]) =>
+  (discoverExternalUrls as jest.Mock).mockResolvedValue(candidates);
+
+beforeAll(async () => {
+  con = await createOrGetConnection();
+});
+
+beforeEach(async () => {
+  jest.restoreAllMocks();
+  jest.clearAllMocks();
+  await saveFixtures(con, User, usersFixture);
+  await saveFixtures(con, Source, sourcesFixture);
+  await saveFixtures(con, ArticlePost, postsFixture);
+  await con.getRepository(AgentSource).save({
+    id: 'asrc-d',
+    name: 'agent source',
+    handle: 'agent-asrc-d',
+    private: true,
+  });
+  await con.getRepository(UserInterest).save({
+    id: 'uir-d',
+    userId: '1',
+    query: 'cool zig projects',
+    status: UserInterestStatus.Active,
+    sourceId: 'asrc-d',
+    fomoThreshold: 0.5,
+  });
+});
+
+afterEach(() => {
+  remoteConfig.vars.interestAgentMaxDiscoveriesPerDay = undefined;
+});
+
+describe('discoverAndIngestExternal', () => {
+  it('creates an article, a share in the agent source, and a discovery finding pointing at the share', async () => {
+    setCandidates([
+      { url: 'https://ext.com/a', title: 'A', rationale: 'why', score: 0.9 },
+    ]);
+
+    const result = await discoverAndIngestExternal({
+      con,
+      logger,
+      interest,
+      query: 'zig',
+    });
+
+    expect(result.added).toBe(1);
+
+    const findings = await con
+      .getRepository(InterestFinding)
+      .findBy({ interestId: 'uir-d' });
+    expect(findings).toHaveLength(1);
+    expect(findings[0].origin).toBe(InterestFindingOrigin.Discovery);
+
+    const share = await con
+      .getRepository(SharePost)
+      .findOneByOrFail({ id: findings[0].postId });
+    expect(share.sourceId).toBe('asrc-d');
+    expect(share.private).toBe(true);
+    expect(share.visible).toBe(true);
+
+    const article = await con
+      .getRepository(ArticlePost)
+      .findOneByOrFail({ id: share.sharedPostId as string });
+    expect(article.url).toBe('https://ext.com/a');
+  });
+
+  it('does not inflate the count when the same url is rediscovered (dedup, no duplicate share/finding)', async () => {
+    setCandidates([
+      { url: 'https://ext.com/a', title: 'A', rationale: 'why', score: 0.9 },
+    ]);
+
+    const first = await discoverAndIngestExternal({
+      con,
+      logger,
+      interest,
+      query: 'zig',
+    });
+    const second = await discoverAndIngestExternal({
+      con,
+      logger,
+      interest,
+      query: 'zig',
+    });
+
+    expect(first.added).toBe(1);
+    expect(second.added).toBe(0);
+    expect(
+      await con.getRepository(InterestFinding).countBy({ interestId: 'uir-d' }),
+    ).toBe(1);
+    expect(
+      await con.getRepository(SharePost).countBy({ sourceId: 'asrc-d' }),
+    ).toBe(1);
+  });
+
+  it('does nothing and does not search when the web source is off', async () => {
+    setCandidates([
+      { url: 'https://ext.com/x', title: 'X', rationale: 'why', score: 0.9 },
+    ]);
+
+    const result = await discoverAndIngestExternal({
+      con,
+      logger,
+      interest: {
+        ...interest,
+        sources: { dailyDev: true, web: false, github: false },
+      },
+      query: 'zig',
+    });
+
+    expect(result.added).toBe(0);
+    expect(discoverExternalUrls).not.toHaveBeenCalled();
+    expect(
+      await con.getRepository(InterestFinding).countBy({ interestId: 'uir-d' }),
+    ).toBe(0);
+  });
+
+  it('skips candidates below the fomo threshold', async () => {
+    setCandidates([
+      { url: 'https://ext.com/b', title: 'B', rationale: 'why', score: 0.2 },
+    ]);
+
+    const result = await discoverAndIngestExternal({
+      con,
+      logger,
+      interest,
+      query: 'zig',
+    });
+
+    expect(result.added).toBe(0);
+    expect(
+      await con.getRepository(InterestFinding).countBy({ interestId: 'uir-d' }),
+    ).toBe(0);
+  });
+
+  it('short-circuits when the daily discovery cap is reached and does not search', async () => {
+    remoteConfig.vars.interestAgentMaxDiscoveriesPerDay = 1;
+
+    await con.getRepository(InterestFinding).save({
+      id: 'finding-cap',
+      interestId: 'uir-d',
+      postId: postsFixture[0].id,
+      score: 0.9,
+      status: InterestFindingStatus.New,
+      origin: InterestFindingOrigin.Discovery,
+    });
+    setCandidates([
+      { url: 'https://ext.com/c', title: 'C', rationale: 'why', score: 0.9 },
+    ]);
+
+    const result = await discoverAndIngestExternal({
+      con,
+      logger,
+      interest,
+      query: 'zig',
+    });
+
+    expect(result.added).toBe(0);
+    expect(discoverExternalUrls).not.toHaveBeenCalled();
+  });
+});

__tests__/common/interest/discoverExternalUrls.ts

@@ -0,0 +1,117 @@
+import type { FastifyBaseLogger } from 'fastify';
+import { anthropicClient } from '../../../src/integrations/anthropic/client';
+import { discoverExternalUrls } from '../../../src/common/interest/discoverExternalUrls';
+
+const logger = {
+  child: () => ({ info: jest.fn(), warn: jest.fn() }),
+} as unknown as FastifyBaseLogger;
+
+const interest = {
+  id: 'i1',
+  query: 'cool zig projects',
+};
+
+const mockContent = (text: string) =>
+  jest
+    .spyOn(anthropicClient, 'createMessage')
+    .mockResolvedValue({ content: [{ type: 'text', text }] } as never);
+
+afterEach(() => {
+  jest.restoreAllMocks();
+});
+
+describe('discoverExternalUrls', () => {
+  it('parses a JSON array of candidates from the model reply', async () => {
+    mockContent(
+      '[{"url":"https://a.com","title":"A","rationale":"r","score":0.9}]',
+    );
+    const result = await discoverExternalUrls({
+      interest,
+      query: 'zig',
+      logger,
+    });
+    expect(result).toEqual([
+      { url: 'https://a.com', title: 'A', rationale: 'r', score: 0.9 },
+    ]);
+  });
+
+  it('drops non-http entries and de-duplicates urls', async () => {
+    mockContent(
+      JSON.stringify([
+        { url: 'https://a.com', title: 'A', rationale: 'r', score: 0.8 },
+        { url: 'https://a.com', title: 'dup', rationale: 'r', score: 0.8 },
+        { url: 'ftp://b.com', title: 'B', rationale: 'r', score: 0.8 },
+        { url: 42, title: 'C', rationale: 'r', score: 0.8 },
+      ]),
+    );
+    const result = await discoverExternalUrls({
+      interest,
+      query: 'zig',
+      logger,
+    });
+    expect(result.map((r) => r.url)).toEqual(['https://a.com']);
+  });
+
+  it('drops daily.dev urls so the agent cannot ingest our own content', async () => {
+    mockContent(
+      JSON.stringify([
+        {
+          url: 'https://daily.dev/posts/x',
+          title: 'own',
+          rationale: 'r',
+          score: 0.9,
+        },
+        {
+          url: 'https://app.daily.dev/posts/y',
+          title: 'own2',
+          rationale: 'r',
+          score: 0.9,
+        },
+        {
+          url: 'https://external.com/a',
+          title: 'ext',
+          rationale: 'r',
+          score: 0.9,
+        },
+      ]),
+    );
+    const result = await discoverExternalUrls({
+      interest,
+      query: 'zig',
+      logger,
+    });
+    expect(result.map((r) => r.url)).toEqual(['https://external.com/a']);
+  });
+
+  it('returns an empty array when the reply is not a JSON array', async () => {
+    mockContent('I could not find anything relevant.');
+    const result = await discoverExternalUrls({
+      interest,
+      query: 'zig',
+      logger,
+    });
+    expect(result).toEqual([]);
+  });
+
+  it('returns an empty array (does not throw) when the client fails', async () => {
+    jest
+      .spyOn(anthropicClient, 'createMessage')
+      .mockRejectedValue(new Error('boom'));
+    const result = await discoverExternalUrls({
+      interest,
+      query: 'zig',
+      logger,
+    });
+    expect(result).toEqual([]);
+  });
+
+  it('issues a generic web_search tool with no domain restriction', async () => {
+    const spy = mockContent('[]');
+    await discoverExternalUrls({ interest, query: 'zig', logger });
+    const request = spy.mock.calls[0][0] as {
+      tools: Array<{ name?: string; allowed_domains?: string[] }>;
+    };
+    expect(request.tools[0].name).toBe('web_search');
+    expect(request.tools[0].allowed_domains).toBeUndefined();
+  });
+});

__tests__/common/interest/runInterestAgent.ts

@@ -33,4 +33,40 @@ describe('getInterestAgentTools', () => {
     expect(tools).toContain('add_to_feed');
     expect(tools).not.toContain('write_post');
   });
+
+  it('omits discover_external when the web source is off', () => {
+    const tools = getInterestAgentTools(undefined, {
+      dailyDev: true,
+      web: false,
+      github: false,
+    });
+    expect(tools).not.toContain('discover_external');
+  });
+
+  it('includes discover_external when the web source is on', () => {
+    expect(
+      getInterestAgentTools(undefined, {
+        dailyDev: true,
+        web: true,
+        github: false,
+      }),
+    ).toContain('discover_external');
+  });
+
+  it('does not enable discover_external for the github source alone (reserved for a future github tool)', () => {
+    const tools = getInterestAgentTools(undefined, {
+      dailyDev: true,
+      web: false,
+      github: true,
+    });
+    expect(tools).not.toContain('discover_external');
+  });
+
+  it('omits discover_external when feed output is off even with web source on', () => {
+    const tools = getInterestAgentTools(
+      { feed: false, post: true, digest: false, notification: true },
+      { dailyDev: true, web: true, github: false },
+    );
+    expect(tools).not.toContain('discover_external');
+  });
 });

__tests__/interests.ts

@@ -11,6 +11,7 @@ import {
   testQueryErrorCode,
 } from './helpers';
 import { ArticlePost, Source, User } from '../src/entity';
+import { FreeformPost } from '../src/entity/posts/FreeformPost';
 import { Feed, FeedOrigin } from '../src/entity/Feed';
 import { AgentSource, SourceType, SourceUser } from '../src/entity/Source';
 import { UserInterest, UserInterestStatus } from '../src/entity/UserInterest';
@@ -375,7 +376,7 @@ describe('mutation updateInterest', () => {
       query: 'cool zig projects',
       status: UserInterestStatus.Active,
       fomoThreshold: 0.5,
-      sources: { dailyDev: true, web: false, github: false },
+      sources: { dailyDev: true, web: true, github: false },
       outputModes: {
         feed: true,
         post: true,
@@ -540,12 +541,12 @@ describe('query interestPosts', () => {
       status: UserInterestStatus.Active,
       sourceId: 'isrc-1',
     });
-    await saveFixtures(con, ArticlePost, [
+    await saveFixtures(con, FreeformPost, [
       {
         id: 'ipost-1',
         shortId: 'ipost-1',
         title: 'Interest summary',
-        url: 'http://interest.com/1',
+        content: 'Interest summary content',
         sourceId: 'isrc-1',
         private: true,
         showOnFeed: false,