diff --git a/apps/api/.env.example b/apps/api/.env.example index 19fe3dc066..8913358e40 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -48,8 +48,28 @@ OPENAI_API_KEY= ANTHROPIC_API_KEY= GROQ_API_KEY= -# 1Password service account token for browser-automation credential storage. -# Optional: when unset, browser automations fall back to human re-authentication. +# Browserbase — the cloud browser that powers Browser Automations. +# REQUIRED for the browser-automation feature (connect, analyze, runs). +BROWSERBASE_API_KEY= +BROWSERBASE_PROJECT_ID= +# Optional model overrides (defaults are sensible; no need to set these). +# BROWSERBASE_CUA_MODEL: screenshot-based navigation agent. Defaults to +# "openai/gpt-5.6-terra" (needs OPENAI_API_KEY; falls back to Claude if it's +# missing). Set to "openai/gpt-5.6-sol" for max accuracy, or an "anthropic/…" +# computer-use model to use Claude instead. +# BROWSERBASE_STAGEHAND_MODEL: model behind page reading / pass-fail verdicts — +# keep this a general model (Claude), NOT a computer-use model. +BROWSERBASE_CUA_MODEL= +BROWSERBASE_STAGEHAND_MODEL= + +# 1Password service account token (starts with "ops_") for storing browser- +# automation logins in a Comp-managed vault. +# Enables the "Sign in for me" password path: Comp stores the login and signs in +# on its own (including generating 2FA codes) for the first sign-in and for +# scheduled runs. +# Optional — when unset, that path is unavailable and users sign in manually in +# the live browser each time (no unattended re-login). +# Create one at 1Password → Developer → Service Accounts (needs vault-create access). OP_SERVICE_ACCOUNT_TOKEN= # Inference.net Catalyst tracing (optional — no-op if CATALYST_OTLP_TOKEN is unset) diff --git a/apps/api/src/browserbase/browser-auth-profile.service.spec.ts b/apps/api/src/browserbase/browser-auth-profile.service.spec.ts index 64470d6be0..1d3c0167e3 100644 --- a/apps/api/src/browserbase/browser-auth-profile.service.spec.ts +++ b/apps/api/src/browserbase/browser-auth-profile.service.spec.ts @@ -1,3 +1,4 @@ +import { ForbiddenException } from '@nestjs/common'; import { BrowserbaseSessionService } from './browserbase-session.service'; import { BrowserAuthProfileService } from './browser-auth-profile.service'; @@ -10,15 +11,21 @@ jest.mock('@db', () => ({ findFirst: jest.fn(), create: jest.fn(), update: jest.fn(), + updateMany: jest.fn(), + delete: jest.fn(), deleteMany: jest.fn(), }, browserbaseContext: { findUnique: jest.fn(), + findFirst: jest.fn(), create: jest.fn(), update: jest.fn(), updateMany: jest.fn(), deleteMany: jest.fn(), }, + browserAutomation: { + findMany: jest.fn(), + }, }, })); @@ -37,6 +44,57 @@ describe('BrowserAuthProfileService', () => { service = new BrowserAuthProfileService(sessions); }); + describe('tenant guards (cross-org IDOR)', () => { + it('assertContextOwnedByOrg passes when a profile in the org owns the context', async () => { + (db.browserAuthProfile.findFirst as jest.Mock).mockResolvedValue({ id: 'bap_1' }); + (db.browserbaseContext.findFirst as jest.Mock).mockResolvedValue(null); + await expect( + service.assertContextOwnedByOrg({ organizationId: 'org_1', contextId: 'ctx_1' }), + ).resolves.toBeUndefined(); + }); + + it('assertContextOwnedByOrg passes when the org-level context row owns it', async () => { + (db.browserAuthProfile.findFirst as jest.Mock).mockResolvedValue(null); + (db.browserbaseContext.findFirst as jest.Mock).mockResolvedValue({ id: 'bbc_1' }); + await expect( + service.assertContextOwnedByOrg({ organizationId: 'org_1', contextId: 'ctx_1' }), + ).resolves.toBeUndefined(); + }); + + it('assertContextOwnedByOrg rejects a context that belongs to another org', async () => { + (db.browserAuthProfile.findFirst as jest.Mock).mockResolvedValue(null); + (db.browserbaseContext.findFirst as jest.Mock).mockResolvedValue(null); + await expect( + service.assertContextOwnedByOrg({ organizationId: 'org_1', contextId: 'ctx_other' }), + ).rejects.toBeInstanceOf(ForbiddenException); + }); + + it('assertSessionOwnedByOrg returns the contextId when the org owns the session', async () => { + jest.spyOn(sessions, 'getSessionContextId').mockResolvedValue('ctx_1'); + (db.browserAuthProfile.findFirst as jest.Mock).mockResolvedValue({ id: 'bap_1' }); + (db.browserbaseContext.findFirst as jest.Mock).mockResolvedValue(null); + await expect( + service.assertSessionOwnedByOrg({ organizationId: 'org_1', sessionId: 'sess_1' }), + ).resolves.toBe('ctx_1'); + }); + + it('assertSessionOwnedByOrg rejects a session whose context is another org', async () => { + jest.spyOn(sessions, 'getSessionContextId').mockResolvedValue('ctx_other'); + (db.browserAuthProfile.findFirst as jest.Mock).mockResolvedValue(null); + (db.browserbaseContext.findFirst as jest.Mock).mockResolvedValue(null); + await expect( + service.assertSessionOwnedByOrg({ organizationId: 'org_1', sessionId: 'sess_x' }), + ).rejects.toBeInstanceOf(ForbiddenException); + }); + + it('assertSessionOwnedByOrg rejects when the session context cannot be resolved', async () => { + jest.spyOn(sessions, 'getSessionContextId').mockResolvedValue(undefined); + await expect( + service.assertSessionOwnedByOrg({ organizationId: 'org_1', sessionId: 'gone' }), + ).rejects.toBeInstanceOf(ForbiddenException); + }); + }); + it('normalizes hostname and login identity when creating a profile', async () => { (db.browserAuthProfile.findUnique as jest.Mock).mockResolvedValue(null); (db.browserbaseContext.findUnique as jest.Mock).mockResolvedValue(null); @@ -188,4 +246,170 @@ describe('BrowserAuthProfileService', () => { where: { organizationId: 'org_1', contextId: '__PENDING__' }, }); }); + + describe('listProfiles', () => { + it('attaches an automation count per connection by hostname', async () => { + (db.browserAuthProfile.findMany as jest.Mock).mockResolvedValue([ + { id: 'bap_gh', organizationId: 'org_1', hostname: 'github.com' }, + { id: 'bap_aws', organizationId: 'org_1', hostname: 'aws.amazon.com' }, + { id: 'bap_dd', organizationId: 'org_1', hostname: 'datadoghq.com' }, + ]); + (db.browserAutomation.findMany as jest.Mock).mockResolvedValue([ + { targetUrl: 'https://github.com/acme/repo' }, + { targetUrl: 'https://github.com/acme/other' }, + { targetUrl: 'https://aws.amazon.com/console' }, + { targetUrl: 'not-a-url' }, // malformed -> skipped, not fatal + ]); + + const result = await service.listProfiles('org_1'); + + expect(db.browserAutomation.findMany).toHaveBeenCalledWith({ + where: { task: { organizationId: 'org_1' } }, + select: { targetUrl: true }, + }); + expect(result.find((p) => p.id === 'bap_gh')?.automationCount).toBe(2); + expect(result.find((p) => p.id === 'bap_aws')?.automationCount).toBe(1); + expect(result.find((p) => p.id === 'bap_dd')?.automationCount).toBe(0); + }); + }); + + describe('updateProfile / deleteProfile', () => { + const existing = { + id: 'bap_1', + organizationId: 'org_1', + hostname: 'app.example.com', + displayName: 'Example', + status: 'verified', + }; + + beforeEach(() => { + (db.browserAuthProfile.update as jest.Mock).mockImplementation( + ({ data }) => ({ ...existing, ...data }), + ); + (db.browserAuthProfile.delete as jest.Mock).mockResolvedValue(existing); + }); + + it('updates only the display name (trimmed)', async () => { + (db.browserAuthProfile.findFirst as jest.Mock).mockResolvedValue(existing); + await service.updateProfile({ + organizationId: 'org_1', + profileId: 'bap_1', + displayName: ' New name ', + }); + expect(db.browserAuthProfile.update).toHaveBeenCalledWith({ + where: { id: 'bap_1' }, + data: { displayName: 'New name' }, + }); + }); + + it('keeps the connection signed in when the URL stays on the same host', async () => { + (db.browserAuthProfile.findFirst as jest.Mock).mockResolvedValue(existing); + await service.updateProfile({ + organizationId: 'org_1', + profileId: 'bap_1', + url: 'https://app.example.com/account/login', + }); + const data = (db.browserAuthProfile.update as jest.Mock).mock.calls[0][0] + .data; + expect(data.lastAuthCheckUrl).toBe( + 'https://app.example.com/account/login', + ); + expect(data.status).toBeUndefined(); + }); + + it('marks needs_reauth when the URL moves to a different host', async () => { + (db.browserAuthProfile.findFirst as jest.Mock).mockResolvedValue(existing); + await service.updateProfile({ + organizationId: 'org_1', + profileId: 'bap_1', + url: 'https://other.com/login', + }); + const data = (db.browserAuthProfile.update as jest.Mock).mock.calls[0][0] + .data; + expect(data.status).toBe('needs_reauth'); + }); + + it('deletes the profile', async () => { + (db.browserAuthProfile.findFirst as jest.Mock).mockResolvedValue(existing); + const result = await service.deleteProfile({ + organizationId: 'org_1', + profileId: 'bap_1', + }); + expect(db.browserAuthProfile.delete).toHaveBeenCalledWith({ + where: { id: 'bap_1' }, + }); + expect(result.success).toBe(true); + }); + + it('throws when updating a missing profile', async () => { + (db.browserAuthProfile.findFirst as jest.Mock).mockResolvedValue(null); + await expect( + service.updateProfile({ + organizationId: 'org_1', + profileId: 'nope', + displayName: 'x', + }), + ).rejects.toThrow('not found'); + }); + }); + + describe('recordSignInAttempt', () => { + it('persists the outcome, detail, and a timestamp, scoped to the org', async () => { + (db.browserAuthProfile.updateMany as jest.Mock).mockResolvedValue({ + count: 1, + }); + + await service.recordSignInAttempt({ + organizationId: 'org_1', + profileId: 'bap_1', + outcome: 'needs_2fa', + detail: 'The account requires a two-factor code to sign in.', + }); + + expect(db.browserAuthProfile.updateMany).toHaveBeenCalledWith({ + where: { id: 'bap_1', organizationId: 'org_1' }, + data: expect.objectContaining({ + lastSignInOutcome: 'needs_2fa', + lastSignInDetail: 'The account requires a two-factor code to sign in.', + lastSignInAt: expect.any(Date), + }), + }); + }); + + it('stores null detail when none is provided', async () => { + (db.browserAuthProfile.updateMany as jest.Mock).mockResolvedValue({ + count: 1, + }); + + await service.recordSignInAttempt({ + organizationId: 'org_1', + profileId: 'bap_1', + outcome: 'logged_in', + }); + + expect(db.browserAuthProfile.updateMany).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + lastSignInOutcome: 'logged_in', + lastSignInDetail: null, + }), + }), + ); + }); + + it('never throws — a logging write failure cannot break sign-in', async () => { + (db.browserAuthProfile.updateMany as jest.Mock).mockRejectedValue( + new Error('db down'), + ); + + await expect( + service.recordSignInAttempt({ + organizationId: 'org_1', + profileId: 'bap_1', + outcome: 'error', + detail: 'boom', + }), + ).resolves.toBeUndefined(); + }); + }); }); diff --git a/apps/api/src/browserbase/browser-auth-profile.service.ts b/apps/api/src/browserbase/browser-auth-profile.service.ts index a21363be13..44d98873af 100644 --- a/apps/api/src/browserbase/browser-auth-profile.service.ts +++ b/apps/api/src/browserbase/browser-auth-profile.service.ts @@ -1,15 +1,20 @@ import { BadRequestException, + ForbiddenException, Injectable, + Logger, NotFoundException, } from '@nestjs/common'; -import { db } from '@db'; +import { db, Prisma } from '@db'; import { defaultProfileDisplayName, normalizeHostnameFromUrl, normalizeLoginIdentity, } from './browserbase-url'; -import { BrowserbaseSessionService } from './browserbase-session.service'; +import { + BrowserbaseSessionService, + INTERACTIVE_VIEWPORT, +} from './browserbase-session.service'; import { BrowserAuthProfileContextService } from './browser-auth-profile-context.service'; import { BrowserbaseOrgContextService, @@ -29,6 +34,8 @@ export interface AuthProfileInput { @Injectable() export class BrowserAuthProfileService { + private readonly logger = new Logger(BrowserAuthProfileService.name); + constructor( private readonly sessions: BrowserbaseSessionService = new BrowserbaseSessionService(), private readonly orgContexts: BrowserbaseOrgContextService = new BrowserbaseOrgContextService( @@ -41,10 +48,33 @@ export class BrowserAuthProfileService { ) {} async listProfiles(organizationId: string) { - return db.browserAuthProfile.findMany({ - where: { organizationId }, - orderBy: [{ hostname: 'asc' }, { updatedAt: 'desc' }], - }); + const [profiles, automations] = await Promise.all([ + db.browserAuthProfile.findMany({ + where: { organizationId }, + orderBy: [{ hostname: 'asc' }, { updatedAt: 'desc' }], + }), + db.browserAutomation.findMany({ + where: { task: { organizationId } }, + select: { targetUrl: true }, + }), + ]); + + // Automations bind to a connection by hostname (not a FK), so tally per host. + const countByHost = new Map(); + for (const { targetUrl } of automations) { + let host: string; + try { + host = normalizeHostnameFromUrl(targetUrl); + } catch { + continue; // skip malformed targetUrls rather than fail the whole list + } + countByHost.set(host, (countByHost.get(host) ?? 0) + 1); + } + + return profiles.map((profile) => ({ + ...profile, + automationCount: countByHost.get(profile.hostname) ?? 0, + })); } async getProfile({ @@ -163,7 +193,12 @@ export class BrowserAuthProfileService { throw new NotFoundException('Browser auth profile not found'); } const readyProfile = await this.profileContexts.ready(profile); - return this.sessions.createSessionWithContext(readyProfile.contextId); + // Human-facing session — use the smaller viewport so the sign-in page reads + // larger in the live view. + return this.sessions.createSessionWithContext( + readyProfile.contextId, + INTERACTIVE_VIEWPORT, + ); } async verifyProfileSession(input: { @@ -224,6 +259,52 @@ export class BrowserAuthProfileService { }); } + async updateProfile(input: { + organizationId: string; + profileId: string; + displayName?: string; + url?: string; + }) { + const profile = await this.getProfile(input); + if (!profile) { + throw new NotFoundException('Browser auth profile not found'); + } + + const data: Prisma.BrowserAuthProfileUpdateInput = {}; + + const name = input.displayName?.trim(); + if (name) data.displayName = name; + + if (input.url !== undefined) { + data.lastAuthCheckUrl = input.url; + // A different hostname means the saved session no longer applies — the + // connection must be re-established. (Hostname is the connection identity, + // so we don't reassign it here.) + try { + if (normalizeHostnameFromUrl(input.url) !== profile.hostname) { + data.status = 'needs_reauth'; + data.blockedReason = 'Sign-in URL changed — reconnect required.'; + } + } catch { + // Ignore an unparseable URL — leave status untouched. + } + } + + return db.browserAuthProfile.update({ + where: { id: profile.id }, + data, + }); + } + + async deleteProfile(input: { organizationId: string; profileId: string }) { + const profile = await this.getProfile(input); + if (!profile) { + throw new NotFoundException('Browser auth profile not found'); + } + await db.browserAuthProfile.delete({ where: { id: profile.id } }); + return { success: true, profile }; + } + async markNeedsReauth(input: { organizationId: string; profileId: string; @@ -262,6 +343,38 @@ export class BrowserAuthProfileService { }); } + /** + * Persist the outcome of the most recent connect/sign-in attempt on the + * connection, so a "can't connect" support ticket can be diagnosed from the + * row alone rather than from ephemeral Trigger.dev logs. Purely diagnostic — + * it does NOT change `status` (the mark* methods own that). Best-effort: scoped + * by org (no cross-tenant write), never throws on a missing row, and swallows + * failures so logging can never break the sign-in flow. + */ + async recordSignInAttempt(input: { + organizationId: string; + profileId: string; + outcome: string; + detail?: string | null; + }): Promise { + try { + await db.browserAuthProfile.updateMany({ + where: { id: input.profileId, organizationId: input.organizationId }, + data: { + lastSignInOutcome: input.outcome, + lastSignInDetail: input.detail ?? null, + lastSignInAt: new Date(), + }, + }); + } catch (error) { + this.logger.warn( + `Failed to record sign-in outcome for profile ${input.profileId}: ${ + error instanceof Error ? error.message : 'unknown error' + }`, + ); + } + } + async getOrCreateOrgContext( organizationId: string, ): Promise<{ contextId: string; isNew: boolean }> { @@ -292,6 +405,57 @@ export class BrowserAuthProfileService { } } + /** + * Tenant guard: a Browserbase context belongs to an org only if a profile (or + * the legacy org-level context row) for that org points at it. Blocks acting on + * another org's context via a raw session/context endpoint (cross-tenant IDOR). + */ + async assertContextOwnedByOrg(input: { + organizationId: string; + contextId: string; + }): Promise { + const [profile, orgContext] = await Promise.all([ + db.browserAuthProfile.findFirst({ + where: { + organizationId: input.organizationId, + contextId: input.contextId, + }, + select: { id: true }, + }), + db.browserbaseContext.findFirst({ + where: { + organizationId: input.organizationId, + contextId: input.contextId, + }, + select: { id: true }, + }), + ]); + if (!profile && !orgContext) { + throw new ForbiddenException( + 'This browser context does not belong to your organization.', + ); + } + } + + /** + * Tenant guard for a live session: resolve the session's context and confirm + * that context belongs to the caller's org. Returns the resolved contextId. + */ + async assertSessionOwnedByOrg(input: { + organizationId: string; + sessionId: string; + }): Promise { + const contextId = await this.sessions.getSessionContextId(input.sessionId); + if (!contextId) { + throw new ForbiddenException('Could not verify the browser session.'); + } + await this.assertContextOwnedByOrg({ + organizationId: input.organizationId, + contextId, + }); + return contextId; + } + private async assertSessionMatchesProfile(input: { sessionId: string; profileContextId: string; diff --git a/apps/api/src/browserbase/browser-auth-profiles.controller.ts b/apps/api/src/browserbase/browser-auth-profiles.controller.ts index 0c4f821a3f..f960e41731 100644 --- a/apps/api/src/browserbase/browser-auth-profiles.controller.ts +++ b/apps/api/src/browserbase/browser-auth-profiles.controller.ts @@ -1,8 +1,20 @@ -import { Body, Controller, Get, Param, Post, UseGuards } from '@nestjs/common'; +import { + BadRequestException, + Body, + Controller, + Delete, + Get, + Param, + Patch, + Post, + Query, + UseGuards, +} from '@nestjs/common'; import { ApiBody, ApiOperation, ApiParam, + ApiQuery, ApiResponse, ApiSecurity, ApiTags, @@ -11,6 +23,11 @@ import { OrganizationId } from '../auth/auth-context.decorator'; import { HybridAuthGuard } from '../auth/hybrid-auth.guard'; import { PermissionGuard } from '../auth/permission.guard'; import { RequirePermission } from '../auth/require-permission.decorator'; +import { BrowserCredentialStorageService } from './browser-credential-storage.service'; +import { + BrowserMfaInstructionsService, + type MfaInstructions, +} from './browser-mfa-instructions.service'; import { BrowserbaseService } from './browserbase.service'; import { BrowserAuthProfileResponseDto, @@ -18,7 +35,11 @@ import { ResolveAuthProfileDto, ResolveAuthProfileResponseDto, SessionResponseDto, + SetAuthProfileTotpDto, + SignInAuthProfileDto, + SignInAuthProfileResponseDto, StoreAuthProfileCredentialsDto, + UpdateAuthProfileDto, VerifyAuthProfileResponseDto, VerifyAuthProfileSessionDto, } from './dto/browserbase.dto'; @@ -28,7 +49,32 @@ import { @UseGuards(HybridAuthGuard, PermissionGuard) @ApiSecurity('apikey') export class BrowserAuthProfilesController { - constructor(private readonly browserbaseService: BrowserbaseService) {} + constructor( + private readonly browserbaseService: BrowserbaseService, + private readonly mfaInstructionsService: BrowserMfaInstructionsService, + private readonly credentialStorageService: BrowserCredentialStorageService, + ) {} + + @Get('mfa-instructions') + @RequirePermission('integration', 'read') + @ApiOperation({ + summary: 'Get authenticator (2FA) setup instructions for a vendor', + description: + 'Returns per-vendor, human-readable steps for finding the authenticator "setup key" (TOTP seed) so a user can enable unattended 2FA. Steps are AI-generated (no per-vendor hardcode), confidence-gated to a universal fallback, and cached per hostname.', + }) + @ApiQuery({ + name: 'host', + description: 'The vendor sign-in URL or hostname (e.g. github.com).', + }) + @ApiResponse({ status: 200 }) + async getMfaInstructions( + @Query('host') host?: string, + ): Promise { + if (!host?.trim()) { + throw new BadRequestException('host query parameter is required'); + } + return this.mfaInstructionsService.getInstructions(host.trim()); + } @Get('profiles') @RequirePermission('integration', 'read') @@ -134,9 +180,138 @@ export class BrowserAuthProfilesController { username: dto.username, password: dto.password, totpSeed: dto.totpSeed, + extraFields: dto.extraFields, + usernameLabel: dto.usernameLabel, })) as BrowserAuthProfileResponseDto; } + @Get('profiles/:profileId/totp') + @RequirePermission('integration', 'read') + @ApiOperation({ + summary: 'Get automatic-2FA status for a connection', + description: + "Reports whether an authenticator setup key (TOTP seed) is stored for this connection, read live from the vault, so scheduled sign-ins can generate 2FA codes unattended.", + }) + @ApiParam({ name: 'profileId', description: 'Browser auth profile ID' }) + @ApiResponse({ status: 200 }) + async getProfileTotp( + @OrganizationId() organizationId: string, + @Param('profileId') profileId: string, + ): Promise<{ configured: boolean }> { + return this.credentialStorageService.getProfileTotpStatus({ + organizationId, + profileId, + }); + } + + @Post('profiles/:profileId/totp') + @RequirePermission('integration', 'update') + @ApiOperation({ + summary: 'Store an authenticator setup key for a connection', + description: + "Attach or replace the authenticator setup key (TOTP seed) on this connection's stored login, enabling unattended 2FA. Does not require re-entering the username or password.", + }) + @ApiParam({ name: 'profileId', description: 'Browser auth profile ID' }) + @ApiBody({ type: SetAuthProfileTotpDto }) + @ApiResponse({ status: 201 }) + async setProfileTotp( + @OrganizationId() organizationId: string, + @Param('profileId') profileId: string, + @Body() dto: SetAuthProfileTotpDto, + ): Promise<{ configured: boolean }> { + return this.credentialStorageService.setProfileTotp({ + organizationId, + profileId, + totpSeed: dto.totpSeed, + }); + } + + @Delete('profiles/:profileId/totp') + @RequirePermission('integration', 'update') + @ApiOperation({ + summary: 'Turn off automatic 2FA for a connection', + description: + 'Remove the stored authenticator setup key. Scheduled runs pause if the vendor then asks for a code.', + }) + @ApiParam({ name: 'profileId', description: 'Browser auth profile ID' }) + @ApiResponse({ status: 200 }) + async clearProfileTotp( + @OrganizationId() organizationId: string, + @Param('profileId') profileId: string, + ): Promise<{ configured: boolean }> { + return this.credentialStorageService.clearProfileTotp({ + organizationId, + profileId, + }); + } + + @Post('profiles/:profileId/sign-in') + @RequirePermission('integration', 'update') + @ApiOperation({ + summary: 'Automated sign-in for a browser auth profile', + description: + 'Starts a background run that signs in to the vendor using the credentials stored for this profile, so the user does not have to type them into the browser. Returns a run handle to subscribe to; if the automated sign-in cannot complete (CAPTCHA, email/SMS code, SSO), the connect flow falls back to a live browser.', + }) + @ApiParam({ name: 'profileId', description: 'Browser auth profile ID' }) + @ApiBody({ type: SignInAuthProfileDto }) + @ApiResponse({ status: 201, type: SignInAuthProfileResponseDto }) + async signInProfile( + @OrganizationId() organizationId: string, + @Param('profileId') profileId: string, + @Body() dto: SignInAuthProfileDto, + ): Promise { + return this.browserbaseService.signInAuthProfile({ + organizationId, + profileId, + url: dto.url, + mode: dto.mode, + usernameLabel: dto.usernameLabel, + }); + } + + @Patch('profiles/:profileId') + @RequirePermission('integration', 'update') + @ApiOperation({ + summary: 'Update a browser auth profile', + description: + 'Edit a connection’s display name and/or sign-in URL. Changing to a different hostname marks the connection as needing reconnection.', + }) + @ApiParam({ name: 'profileId', description: 'Browser auth profile ID' }) + @ApiBody({ type: UpdateAuthProfileDto }) + @ApiResponse({ status: 200, type: BrowserAuthProfileResponseDto }) + async updateProfile( + @OrganizationId() organizationId: string, + @Param('profileId') profileId: string, + @Body() dto: UpdateAuthProfileDto, + ): Promise { + return (await this.browserbaseService.updateAuthProfile({ + organizationId, + profileId, + displayName: dto.displayName, + url: dto.url, + })) as BrowserAuthProfileResponseDto; + } + + @Delete('profiles/:profileId') + @RequirePermission('integration', 'delete') + @ApiOperation({ + summary: 'Remove a browser auth profile', + description: + 'Delete a connection and best-effort remove its stored login from the vault. Automations that relied on it stop running until reconnected.', + }) + @ApiParam({ name: 'profileId', description: 'Browser auth profile ID' }) + @ApiResponse({ status: 200 }) + async deleteProfile( + @OrganizationId() organizationId: string, + @Param('profileId') profileId: string, + ): Promise<{ success: boolean }> { + const result = await this.browserbaseService.deleteAuthProfile({ + organizationId, + profileId, + }); + return { success: result.success }; + } + @Post('profiles/:profileId/needs-reauth') @RequirePermission('integration', 'update') @ApiOperation({ diff --git a/apps/api/src/browserbase/browser-automation-crud.service.ts b/apps/api/src/browserbase/browser-automation-crud.service.ts index d13d67afc0..28fa699fc1 100644 --- a/apps/api/src/browserbase/browser-automation-crud.service.ts +++ b/apps/api/src/browserbase/browser-automation-crud.service.ts @@ -8,6 +8,53 @@ const normalizeCriteria = (value: string | null | undefined): string | null => { return trimmed.length === 0 ? null : trimmed; }; +/** One step of a (possibly multi-vendor) automation. */ +export interface BrowserAutomationStepInput { + profileId?: string | null; + targetUrl: string; + instruction: string; + evaluationCriteria?: string | null; +} + +/** + * Every automation is stored as an ordered list of steps. When only the legacy + * single instruction is supplied, treat it as a one-step automation. + */ +function resolveSteps(data: { + targetUrl?: string; + instruction?: string; + evaluationCriteria?: string | null; + steps?: BrowserAutomationStepInput[]; +}): BrowserAutomationStepInput[] { + if (data.steps && data.steps.length > 0) return data.steps; + return [ + { + targetUrl: data.targetUrl ?? '', + instruction: data.instruction ?? '', + evaluationCriteria: data.evaluationCriteria, + }, + ]; +} + +const toStepCreate = (steps: BrowserAutomationStepInput[]) => + steps.map((step, index) => ({ + order: index, + profileId: step.profileId ?? null, + targetUrl: step.targetUrl, + instruction: step.instruction, + evaluationCriteria: normalizeCriteria(step.evaluationCriteria), + })); + +const STEP_INCLUDE = { steps: { orderBy: { order: 'asc' as const } } }; + +/** Per-step evidence for a run — one screenshot + verdict per step, in order. */ +const RUN_STEP_RUNS_INCLUDE = { + stepRuns: { + orderBy: { order: 'asc' as const }, + include: { step: { select: { targetUrl: true } } }, + }, +}; + @Injectable() export class BrowserAutomationCrudService { constructor( @@ -22,6 +69,7 @@ export class BrowserAutomationCrudService { targetUrl: string; instruction: string; evaluationCriteria?: string; + steps?: BrowserAutomationStepInput[]; scheduleFrequency?: TaskFrequency; }, organizationId?: string, @@ -30,19 +78,37 @@ export class BrowserAutomationCrudService { await this.requireTaskInOrg({ taskId: data.taskId, organizationId }); } + const steps = resolveSteps(data); + const first = steps[0]; + + // A task's browser evidence shares one cadence (set from the section + // header), so a new automation inherits the task's current schedule rather + // than silently defaulting to daily. + const scheduleFrequency = + data.scheduleFrequency ?? + ( + await db.browserAutomation.findFirst({ + where: { taskId: data.taskId }, + orderBy: { createdAt: 'asc' }, + select: { scheduleFrequency: true }, + }) + )?.scheduleFrequency; + return db.browserAutomation.create({ data: { taskId: data.taskId, name: data.name, description: data.description, - targetUrl: data.targetUrl, - instruction: data.instruction, - evaluationCriteria: normalizeCriteria(data.evaluationCriteria), + // Keep the legacy single fields in sync with the first step until the + // engine reads steps directly (then these columns get dropped). + targetUrl: first.targetUrl, + instruction: first.instruction, + evaluationCriteria: normalizeCriteria(first.evaluationCriteria), isEnabled: true, - ...(data.scheduleFrequency !== undefined - ? { scheduleFrequency: data.scheduleFrequency } - : {}), + ...(scheduleFrequency !== undefined ? { scheduleFrequency } : {}), + steps: { create: toStepCreate(steps) }, }, + include: STEP_INCLUDE, }); } @@ -51,7 +117,12 @@ export class BrowserAutomationCrudService { where: { id: automationId }, include: { task: { select: { organizationId: true } }, - runs: { orderBy: { createdAt: 'desc' }, take: 10 }, + runs: { + orderBy: { createdAt: 'desc' }, + take: 10, + include: RUN_STEP_RUNS_INCLUDE, + }, + ...STEP_INCLUDE, }, }); return this.hideCrossOrgAutomation({ automation, organizationId }); @@ -65,7 +136,12 @@ export class BrowserAutomationCrudService { return db.browserAutomation.findMany({ where: { taskId }, include: { - runs: { orderBy: { createdAt: 'desc' }, take: 1 }, + runs: { + orderBy: { createdAt: 'desc' }, + take: 1, + include: RUN_STEP_RUNS_INCLUDE, + }, + ...STEP_INCLUDE, }, orderBy: { createdAt: 'desc' }, }); @@ -80,6 +156,7 @@ export class BrowserAutomationCrudService { instruction?: string; evaluationCriteria?: string; isEnabled?: boolean; + steps?: BrowserAutomationStepInput[]; scheduleFrequency?: TaskFrequency; }, organizationId?: string, @@ -88,16 +165,58 @@ export class BrowserAutomationCrudService { await this.requireAutomationInOrg({ automationId, organizationId }); } - const { evaluationCriteria, scheduleFrequency, ...rest } = data; - return db.browserAutomation.update({ - where: { id: automationId }, - data: { - ...rest, - ...(evaluationCriteria !== undefined - ? { evaluationCriteria: normalizeCriteria(evaluationCriteria) } - : {}), - ...(scheduleFrequency !== undefined ? { scheduleFrequency } : {}), - }, + const { evaluationCriteria, scheduleFrequency, steps, ...rest } = data; + + // Steps supplied → replace the whole sequence and sync the legacy columns + // from the first step (atomic so a run never sees a half-updated list). + if (steps && steps.length > 0) { + const first = steps[0]; + return db.$transaction(async (tx) => { + await tx.browserAutomationStep.deleteMany({ where: { automationId } }); + return tx.browserAutomation.update({ + where: { id: automationId }, + data: { + ...rest, + targetUrl: first.targetUrl, + instruction: first.instruction, + evaluationCriteria: normalizeCriteria(first.evaluationCriteria), + ...(scheduleFrequency !== undefined ? { scheduleFrequency } : {}), + steps: { create: toStepCreate(steps) }, + }, + include: STEP_INCLUDE, + }); + }); + } + + // Legacy single-field edit → update the automation and mirror it onto the + // first step so the two representations stay consistent. + const stepPatch = { + ...(rest.targetUrl !== undefined ? { targetUrl: rest.targetUrl } : {}), + ...(rest.instruction !== undefined ? { instruction: rest.instruction } : {}), + ...(evaluationCriteria !== undefined + ? { evaluationCriteria: normalizeCriteria(evaluationCriteria) } + : {}), + }; + + return db.$transaction(async (tx) => { + const updated = await tx.browserAutomation.update({ + where: { id: automationId }, + data: { + ...rest, + ...(evaluationCriteria !== undefined + ? { evaluationCriteria: normalizeCriteria(evaluationCriteria) } + : {}), + ...(scheduleFrequency !== undefined ? { scheduleFrequency } : {}), + }, + include: STEP_INCLUDE, + }); + if (Object.keys(stepPatch).length > 0) { + await tx.browserAutomationStep.updateMany({ + where: { automationId, order: 0 }, + data: stepPatch, + }); + } + return updated; }); } @@ -108,37 +227,79 @@ export class BrowserAutomationCrudService { return db.browserAutomation.delete({ where: { id: automationId } }); } + /** + * Set one cadence for every browser automation on a task. Browser evidence + * shares a task-level schedule (there's no need to run different vendors in + * one task on different days), so this bulk-updates them together. + */ + async setTaskSchedule( + taskId: string, + scheduleFrequency: TaskFrequency, + organizationId?: string, + ) { + if (organizationId) { + await this.requireTaskInOrg({ taskId, organizationId }); + } + const { count } = await db.browserAutomation.updateMany({ + where: { taskId }, + data: { scheduleFrequency }, + }); + return { success: true, scheduleFrequency, updated: count }; + } + + /** Presign a run's screenshots (full page + close-up) and each step's. */ + private async presignRun< + T extends { + screenshotUrl: string | null; + focusScreenshotUrl?: string | null; + stepRuns?: Array<{ + screenshotUrl: string | null; + focusScreenshotUrl?: string | null; + }>; + }, + >(run: T): Promise { + const presign = ( + key: string | null | undefined, + ): Promise => + key ? this.screenshots.getPresignedUrl({ key }) : Promise.resolve(key); + const [screenshotUrl, focusScreenshotUrl, stepRuns] = await Promise.all([ + presign(run.screenshotUrl), + presign(run.focusScreenshotUrl), + run.stepRuns + ? Promise.all( + run.stepRuns.map(async (stepRun) => ({ + ...stepRun, + screenshotUrl: await presign(stepRun.screenshotUrl), + focusScreenshotUrl: await presign(stepRun.focusScreenshotUrl), + })), + ) + : Promise.resolve(run.stepRuns), + ]); + return { ...run, screenshotUrl, focusScreenshotUrl, stepRuns } as T; + } + async getRunWithPresignedUrl(runId: string, organizationId?: string) { const run = await db.browserAutomationRun.findUnique({ where: { id: runId }, - include: { automation: { include: { task: true } } }, + include: { + automation: { include: { task: true } }, + ...RUN_STEP_RUNS_INCLUDE, + }, }); if (!run) return null; if (organizationId && run.automation.task.organizationId !== organizationId) { return null; } - if (!run.screenshotUrl) return run; - const screenshotUrl = await this.screenshots.getPresignedUrl({ - key: run.screenshotUrl, - }); - return { ...run, screenshotUrl }; + return this.presignRun(run); } async getAutomationsWithPresignedUrls(taskId: string, organizationId?: string) { const automations = await this.getBrowserAutomationsForTask(taskId, organizationId); return Promise.all( - automations.map(async (automation) => { - const runsWithUrls = await Promise.all( - automation.runs.map(async (run) => { - if (!run.screenshotUrl) return run; - const screenshotUrl = await this.screenshots.getPresignedUrl({ - key: run.screenshotUrl, - }); - return { ...run, screenshotUrl }; - }), - ); - return { ...automation, runs: runsWithUrls }; - }), + automations.map(async (automation) => ({ + ...automation, + runs: await Promise.all(automation.runs.map((run) => this.presignRun(run))), + })), ); } diff --git a/apps/api/src/browserbase/browser-automation-draft.service.spec.ts b/apps/api/src/browserbase/browser-automation-draft.service.spec.ts new file mode 100644 index 0000000000..ba9b29421e --- /dev/null +++ b/apps/api/src/browserbase/browser-automation-draft.service.spec.ts @@ -0,0 +1,81 @@ +jest.mock('@db', () => ({ + db: { + task: { findFirst: jest.fn() }, + browserAutomationDraft: { + findMany: jest.fn(), + findFirst: jest.fn(), + create: jest.fn(), + update: jest.fn(), + delete: jest.fn(), + }, + }, + Prisma: {}, +})); + +import { db } from '@db'; +import { BrowserAutomationDraftService } from './browser-automation-draft.service'; + +describe('BrowserAutomationDraftService', () => { + let service: BrowserAutomationDraftService; + + beforeEach(() => { + jest.clearAllMocks(); + service = new BrowserAutomationDraftService(); + }); + + it('creates a draft after checking the task is in the org, storing plain JSON steps', async () => { + (db.task.findFirst as jest.Mock).mockResolvedValue({ id: 'tsk_1' }); + (db.browserAutomationDraft.create as jest.Mock).mockResolvedValue({ id: 'bad_1' }); + + await service.createDraft( + { + taskId: 'tsk_1', + name: 'GitHub 2FA', + steps: [{ profileId: 'p1', instruction: 'x', extra: undefined }], + }, + 'org_1', + ); + + expect(db.task.findFirst).toHaveBeenCalledWith({ + where: { id: 'tsk_1', organizationId: 'org_1' }, + select: { id: true }, + }); + const data = (db.browserAutomationDraft.create as jest.Mock).mock.calls[0][0].data; + expect(data.taskId).toBe('tsk_1'); + expect(data.name).toBe('GitHub 2FA'); + // toJson strips undefined -> plain JSON. + expect(data.steps).toEqual([{ profileId: 'p1', instruction: 'x' }]); + }); + + it('rejects creating a draft for a task in another org', async () => { + (db.task.findFirst as jest.Mock).mockResolvedValue(null); + + await expect( + service.createDraft({ taskId: 'tsk_x', steps: [] }, 'org_1'), + ).rejects.toThrow('Task not found'); + expect(db.browserAutomationDraft.create).not.toHaveBeenCalled(); + }); + + it('updates a draft only when it belongs to the org', async () => { + (db.browserAutomationDraft.findFirst as jest.Mock).mockResolvedValue({ id: 'bad_1' }); + (db.browserAutomationDraft.update as jest.Mock).mockResolvedValue({ id: 'bad_1' }); + + await service.updateDraft('bad_1', { steps: [{ instruction: 'y' }] }, 'org_1'); + + expect(db.browserAutomationDraft.findFirst).toHaveBeenCalledWith({ + where: { id: 'bad_1', task: { organizationId: 'org_1' } }, + select: { id: true }, + }); + expect(db.browserAutomationDraft.update).toHaveBeenCalledWith({ + where: { id: 'bad_1' }, + data: { steps: [{ instruction: 'y' }] }, + }); + }); + + it('rejects deleting a draft from another org', async () => { + (db.browserAutomationDraft.findFirst as jest.Mock).mockResolvedValue(null); + + await expect(service.deleteDraft('bad_x', 'org_1')).rejects.toThrow('Draft not found'); + expect(db.browserAutomationDraft.delete).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/src/browserbase/browser-automation-draft.service.ts b/apps/api/src/browserbase/browser-automation-draft.service.ts new file mode 100644 index 0000000000..e8f2c80121 --- /dev/null +++ b/apps/api/src/browserbase/browser-automation-draft.service.ts @@ -0,0 +1,80 @@ +import { Injectable, NotFoundException } from '@nestjs/common'; +import { db, Prisma } from '@db'; + +/** Strip class instances / undefined down to plain JSON for a Prisma Json column. */ +function toJson(value: unknown): Prisma.InputJsonValue { + return JSON.parse(JSON.stringify(value ?? null)); +} + +interface DraftInput { + name?: string | null; + steps: unknown; +} + +/** + * Server-side store for in-progress (unsaved) automations. Drafts are scoped to + * a task (and therefore an org); they hold the composer's raw step state as JSON + * and are deleted once the automation is saved for real. + */ +@Injectable() +export class BrowserAutomationDraftService { + private async requireTaskInOrg(taskId: string, organizationId: string) { + const task = await db.task.findFirst({ + where: { id: taskId, organizationId }, + select: { id: true }, + }); + if (!task) throw new NotFoundException('Task not found'); + } + + private async requireDraftInOrg(draftId: string, organizationId: string) { + const draft = await db.browserAutomationDraft.findFirst({ + where: { id: draftId, task: { organizationId } }, + select: { id: true }, + }); + if (!draft) throw new NotFoundException('Draft not found'); + } + + async listDraftsForTask(taskId: string, organizationId: string) { + await this.requireTaskInOrg(taskId, organizationId); + return db.browserAutomationDraft.findMany({ + where: { taskId }, + orderBy: { updatedAt: 'desc' }, + }); + } + + async createDraft( + input: DraftInput & { taskId: string; createdById?: string | null }, + organizationId: string, + ) { + await this.requireTaskInOrg(input.taskId, organizationId); + return db.browserAutomationDraft.create({ + data: { + taskId: input.taskId, + createdById: input.createdById ?? null, + name: input.name ?? null, + steps: toJson(input.steps), + }, + }); + } + + async updateDraft( + draftId: string, + input: { name?: string | null; steps?: unknown }, + organizationId: string, + ) { + await this.requireDraftInOrg(draftId, organizationId); + return db.browserAutomationDraft.update({ + where: { id: draftId }, + data: { + ...(input.name !== undefined ? { name: input.name } : {}), + ...(input.steps !== undefined ? { steps: toJson(input.steps) } : {}), + }, + }); + } + + async deleteDraft(draftId: string, organizationId: string) { + await this.requireDraftInOrg(draftId, organizationId); + await db.browserAutomationDraft.delete({ where: { id: draftId } }); + return { success: true }; + } +} diff --git a/apps/api/src/browserbase/browser-automation-execution.service.spec.ts b/apps/api/src/browserbase/browser-automation-execution.service.spec.ts index 67e0acfa31..88dc327485 100644 --- a/apps/api/src/browserbase/browser-automation-execution.service.spec.ts +++ b/apps/api/src/browserbase/browser-automation-execution.service.spec.ts @@ -9,6 +9,7 @@ jest.mock('@db', () => ({ $transaction: jest.fn(), browserAutomation: { findUnique: jest.fn() }, browserAutomationRun: { updateMany: jest.fn(), findUnique: jest.fn() }, + browserAutomationStepRun: { create: jest.fn(), update: jest.fn() }, }, Prisma: { TransactionIsolationLevel: { Serializable: 'Serializable' }, @@ -43,6 +44,9 @@ describe('BrowserAutomationExecutionService', () => { (db.browserAutomationRun.updateMany as jest.Mock).mockResolvedValue({ count: 1, }); + (db.browserAutomationStepRun.create as jest.Mock).mockResolvedValue({ + id: 'basr_1', + }); }); it('persists a failed terminal state when the runner throws', async () => { @@ -56,11 +60,15 @@ describe('BrowserAutomationExecutionService', () => { hostname: 'example.com', loginIdentity: '', displayName: 'example.com browser profile', + identifierLabel: null, contextId: 'ctx_1', status: 'verified', lastVerifiedAt: null, lastAuthCheckUrl: null, blockedReason: null, + lastSignInOutcome: null, + lastSignInDetail: null, + lastSignInAt: null, vaultProvider: null, vaultExternalItemRef: null, vaultConnectionId: null, diff --git a/apps/api/src/browserbase/browser-automation-execution.service.ts b/apps/api/src/browserbase/browser-automation-execution.service.ts index fb752d0802..7bf3155b49 100644 --- a/apps/api/src/browserbase/browser-automation-execution.service.ts +++ b/apps/api/src/browserbase/browser-automation-execution.service.ts @@ -6,8 +6,15 @@ import { } from '@nestjs/common'; import { db } from '@db'; import { BrowserAuthProfileService } from './browser-auth-profile.service'; -import { BrowserAutomationRunStoreService } from './browser-automation-run-store.service'; import { failedBrowserEvidenceRunResult } from './browser-automation-run-result'; +import { BrowserAutomationRunStoreService } from './browser-automation-run-store.service'; +import { stepsForRun } from './browser-automation-step-results'; +import { BrowserAutomationStepRunnerService } from './browser-automation-step-runner.service'; +import { + createEvidenceTimeline, + type BrowserRunLivePhase, + type EvidenceTimelineStep, +} from './browser-evidence-step-timeline'; import { BrowserEvidenceRunnerService, type BrowserEvidenceRunResult, @@ -17,6 +24,7 @@ import { BrowserbaseSessionService } from './browserbase-session.service'; @Injectable() export class BrowserAutomationExecutionService { private readonly logger = new Logger(BrowserAutomationExecutionService.name); + private readonly stepRunner: BrowserAutomationStepRunnerService; constructor( private readonly sessions: BrowserbaseSessionService = new BrowserbaseSessionService(), @@ -27,7 +35,15 @@ export class BrowserAutomationExecutionService { sessions, ), private readonly runs: BrowserAutomationRunStoreService = new BrowserAutomationRunStoreService(), - ) {} + ) { + // Built here (not injected) so it reuses these exact profile/runner/run + // instances — including the spied ones in unit tests. + this.stepRunner = new BrowserAutomationStepRunnerService( + this.profiles, + this.runner, + this.runs, + ); + } async startAutomationWithLiveView( automationId: string, @@ -57,7 +73,7 @@ export class BrowserAutomationExecutionService { startedAt: run.startedAt, result, }); - await this.applyProfileResult({ + await this.stepRunner.applyProfileResult({ organizationId, profileId: profile.id, result, @@ -71,6 +87,8 @@ export class BrowserAutomationExecutionService { runId: string, sessionId: string, organizationId: string, + /** Live activity timeline, surfaced to the Run live view via realtime. */ + onSteps?: (steps: EvidenceTimelineStep[]) => void, ) { const automation = await this.getRunnableAutomation({ automationId, @@ -86,6 +104,7 @@ export class BrowserAutomationExecutionService { targetUrl: automation.targetUrl, profileId: run.profileId ?? undefined, }); + const timeline = createEvidenceTimeline(onSteps); let result: BrowserEvidenceRunResult; try { result = await this.runner.executeEvidenceOnSession({ @@ -105,6 +124,7 @@ export class BrowserAutomationExecutionService { vaultExternalItemRef: profile.vaultExternalItemRef, vaultConnectionId: profile.vaultConnectionId, }, + onLog: (entry) => timeline.step(entry.message), beforeExecution: () => this.runs.assertRunIsStillActive({ runId, automationId }), }); @@ -114,8 +134,16 @@ export class BrowserAutomationExecutionService { result = failedBrowserEvidenceRunResult(error); } + timeline.finish( + result.success + ? 'done' + : result.status === 'blocked' || result.needsReauth + ? 'warn' + : 'fail', + ); + await this.runs.finishRun({ runId, startedAt: run.startedAt, result }); - await this.applyProfileResult({ + await this.stepRunner.applyProfileResult({ organizationId, profileId: profile.id, result, @@ -123,125 +151,78 @@ export class BrowserAutomationExecutionService { return this.toRunResponse({ runId, result }); } - async runBrowserAutomation(automationId: string, organizationId: string) { + /** + * The interactive "Run" — executes the FULL step sequence (every vendor), with + * step 0 running on the pre-opened live session (so it's watchable) and later + * vendors each in their own session. Streams a combined step timeline. This is + * what start-live + execute-live drives, so clicking Run no longer stops after + * the first vendor. + */ + async executeAutomationLive( + automationId: string, + runId: string, + sessionId: string, + organizationId: string, + onSteps?: (steps: EvidenceTimelineStep[]) => void, + onLiveView?: (url: string) => void, + onLivePhase?: (phase: BrowserRunLivePhase) => void, + ) { const automation = await this.getRunnableAutomation({ automationId, organizationId, }); - const profile = await this.profiles.resolveProfileForTarget({ + const run = await this.runs.getActiveRun({ runId, automationId }); + const steps = stepsForRun(automation); + const firstProfile = await this.stepRunner.resolveStepProfile({ organizationId, - targetUrl: automation.targetUrl, + step: steps[0], }); - const run = await this.runs.createRun({ + + const result = await this.stepRunner.runSteps({ + organizationId, + taskId: automation.taskId, automationId, - profileId: profile.id, + runId, + steps, + firstProfile, + firstSessionId: sessionId, + onSteps, + onLiveView, + onLivePhase, }); - // A profile with stored credentials can recover an expired session on its - // own, so let a needs_reauth profile attempt the run instead of blocking it. - // Profiles that are blocked or never verified still require a human. - const canAutoRelogin = - profile.status === 'needs_reauth' && Boolean(profile.vaultProvider); - if (profile.status !== 'verified' && !canAutoRelogin) { - const result = this.profileBlockedResult(profile.status); - await this.runs.finishRun({ - runId: run.id, - startedAt: run.startedAt, - result, - }); - return this.toRunResponse({ runId: run.id, result }); - } + await this.runs.finishRun({ runId, startedAt: run.startedAt, result }); + return this.toRunResponse({ runId, result }); + } - let result: BrowserEvidenceRunResult; - try { - result = await this.runner.runEvidence({ - organizationId, - taskId: automation.taskId, - automationId, - runId: run.id, - targetUrl: automation.targetUrl, - instruction: automation.instruction, - evaluationCriteria: automation.evaluationCriteria, - profile: { - id: profile.id, - hostname: profile.hostname, - contextId: profile.contextId, - vaultProvider: profile.vaultProvider, - vaultExternalItemRef: profile.vaultExternalItemRef, - vaultConnectionId: profile.vaultConnectionId, - }, - }); - } catch (error) { - this.logger.error('Browser evidence runner failed', error); - result = failedBrowserEvidenceRunResult(error); - } - await this.runs.finishRun({ - runId: run.id, - startedAt: run.startedAt, - result, - }); - await this.applyProfileResult({ + async runBrowserAutomation(automationId: string, organizationId: string) { + const automation = await this.getRunnableAutomation({ + automationId, organizationId, - profileId: profile.id, - result, }); - return this.toRunResponse({ runId: run.id, result }); - } + const steps = stepsForRun(automation); - private async applyProfileResult(input: { - organizationId: string; - profileId: string; - result: BrowserEvidenceRunResult; - }) { - // A successful run proves the session is valid again — clear any prior - // needs_reauth/blocked state (e.g. after a credential-backed auto sign-in). - if (input.result.success) { - await this.profiles.markVerified({ - organizationId: input.organizationId, - profileId: input.profileId, - }); - return; - } - - if (input.result.failureCode === 'needs_reauth') { - await this.profiles.markNeedsReauth({ - organizationId: input.organizationId, - profileId: input.profileId, - reason: input.result.blockedReason, - }); - } + // Attribute the run to the first step's connection. + const firstProfile = await this.stepRunner.resolveStepProfile({ + organizationId, + step: steps[0], + }); + const run = await this.runs.createRun({ + automationId, + profileId: firstProfile?.id, + }); - if ( - input.result.failureCode === 'captcha_blocked' || - input.result.failureCode === 'needs_user_action' - ) { - await this.profiles.markBlocked({ - organizationId: input.organizationId, - profileId: input.profileId, - reason: - input.result.blockedReason ?? - input.result.error ?? - 'User action is required before this automation can run.', - }); - } - } + const result = await this.stepRunner.runSteps({ + organizationId, + taskId: automation.taskId, + automationId, + runId: run.id, + steps, + firstProfile, + }); - private profileBlockedResult(status: string): BrowserEvidenceRunResult { - const needsUserAction = status === 'blocked'; - return { - success: false, - status: 'blocked', - error: needsUserAction - ? 'This browser profile is blocked. Resolve the blocked state before running automations.' - : 'This browser profile is not verified. Reconnect it before running automations.', - needsReauth: !needsUserAction, - failureCode: needsUserAction ? 'needs_user_action' : 'needs_reauth', - failureStage: 'auth', - blockedReason: needsUserAction - ? 'Browser profile is blocked.' - : 'Browser profile is not verified.', - logs: [], - }; + await this.runs.finishRun({ runId: run.id, startedAt: run.startedAt, result }); + return this.toRunResponse({ runId: run.id, result }); } private toRunResponse(input: { @@ -272,6 +253,7 @@ export class BrowserAutomationExecutionService { task: { select: { title: true, description: true, organizationId: true }, }, + steps: { orderBy: { order: 'asc' } }, }, }); if ( diff --git a/apps/api/src/browserbase/browser-automation-run-store.service.ts b/apps/api/src/browserbase/browser-automation-run-store.service.ts index ac02332299..124d5b4308 100644 --- a/apps/api/src/browserbase/browser-automation-run-store.service.ts +++ b/apps/api/src/browserbase/browser-automation-run-store.service.ts @@ -58,6 +58,7 @@ export class BrowserAutomationRunStoreService { ? Date.now() - input.startedAt.getTime() : 0, screenshotUrl: input.result.screenshotKey, + focusScreenshotUrl: input.result.focusScreenshotKey ?? null, evaluationStatus: input.result.evaluationStatus ?? null, evaluationReason: input.result.evaluationReason ?? null, error: input.result.error, @@ -74,6 +75,40 @@ export class BrowserAutomationRunStoreService { } } + async createStepRun(input: { + runId: string; + stepId: string | null; + order: number; + }) { + return db.browserAutomationStepRun.create({ + data: { + runId: input.runId, + stepId: input.stepId, + order: input.order, + status: 'running', + startedAt: new Date(), + }, + }); + } + + async finishStepRun(input: { + stepRunId: string; + result: BrowserEvidenceRunResult; + }): Promise { + await db.browserAutomationStepRun.update({ + where: { id: input.stepRunId }, + data: { + status: input.result.status, + completedAt: new Date(), + screenshotUrl: input.result.screenshotKey ?? null, + focusScreenshotUrl: input.result.focusScreenshotKey ?? null, + evaluationStatus: input.result.evaluationStatus ?? null, + evaluationReason: input.result.evaluationReason ?? null, + error: input.result.error ?? null, + }, + }); + } + async getActiveRun(input: { runId: string; automationId: string }) { const run = await db.browserAutomationRun.findUnique({ where: { id: input.runId }, diff --git a/apps/api/src/browserbase/browser-automation-step-results.ts b/apps/api/src/browserbase/browser-automation-step-results.ts new file mode 100644 index 0000000000..93d9540453 --- /dev/null +++ b/apps/api/src/browserbase/browser-automation-step-results.ts @@ -0,0 +1,129 @@ +import type { BrowserEvidenceRunResult } from './browser-evidence-runner.service'; + +/** A normalized step to execute (real step row, or a legacy inline instruction). */ +export type StepForRun = { + id: string | null; + order: number; + profileId: string | null; + targetUrl: string; + instruction: string; + evaluationCriteria: string | null; +}; + +/** Steps to run: the ordered step rows, or the inline instruction as one step. */ +export function stepsForRun(automation: { + targetUrl: string; + instruction: string; + evaluationCriteria: string | null; + steps?: Array<{ + id: string; + order: number; + profileId: string | null; + targetUrl: string; + instruction: string; + evaluationCriteria: string | null; + }>; +}): StepForRun[] { + if (automation.steps && automation.steps.length > 0) { + return [...automation.steps] + .sort((a, b) => a.order - b.order) + .map((step) => ({ + id: step.id, + order: step.order, + profileId: step.profileId, + targetUrl: step.targetUrl, + instruction: step.instruction, + evaluationCriteria: step.evaluationCriteria, + })); + } + return [ + { + id: null, + order: 0, + profileId: null, + targetUrl: automation.targetUrl, + instruction: automation.instruction, + evaluationCriteria: automation.evaluationCriteria, + }, + ]; +} + +export function profileMissingResult(): BrowserEvidenceRunResult { + return { + success: false, + status: 'blocked', + error: 'This step has no connected vendor login. Connect one, then run again.', + needsReauth: true, + failureCode: 'needs_reauth', + failureStage: 'auth', + blockedReason: 'No connection is bound to this step.', + logs: [], + }; +} + +export function profileBlockedResult(status: string): BrowserEvidenceRunResult { + const needsUserAction = status === 'blocked'; + return { + success: false, + status: 'blocked', + error: needsUserAction + ? 'This browser profile is blocked. Resolve the blocked state before running automations.' + : 'This browser profile is not verified. Reconnect it before running automations.', + needsReauth: !needsUserAction, + failureCode: needsUserAction ? 'needs_user_action' : 'needs_reauth', + failureStage: 'auth', + blockedReason: needsUserAction + ? 'Browser profile is blocked.' + : 'Browser profile is not verified.', + logs: [], + }; +} + +/** + * Combine per-step results into one run verdict: overall success only if every + * step ran; the check fails if any step's check fails; error metadata comes from + * the first step that failed technically. A single step passes through verbatim. + */ +export function rollUpStepResults( + results: BrowserEvidenceRunResult[], +): BrowserEvidenceRunResult { + if (results.length === 1) return results[0]; + + const firstProblem = results.find((result) => !result.success); + const failedCheck = results.find( + (result) => result.evaluationStatus === 'fail', + ); + const lastWithShot = [...results] + .reverse() + .find((result) => result.screenshotKey); + + const status: BrowserEvidenceRunResult['status'] = results.every( + (result) => result.success, + ) + ? 'completed' + : results.some((result) => result.status === 'failed') + ? 'failed' + : 'blocked'; + + return { + success: results.every((result) => result.success), + status, + screenshotKey: lastWithShot?.screenshotKey, + screenshotUrl: lastWithShot?.screenshotUrl, + finalUrl: results[results.length - 1]?.finalUrl, + evaluationStatus: failedCheck + ? 'fail' + : results.some((result) => result.evaluationStatus === 'pass') + ? 'pass' + : undefined, + evaluationReason: failedCheck?.evaluationReason, + error: firstProblem?.error, + needsReauth: results.some((result) => result.needsReauth), + failureCode: firstProblem?.failureCode, + failureStage: firstProblem?.failureStage, + blockedReason: firstProblem?.blockedReason, + logs: results.flatMap((result) => + Array.isArray(result.logs) ? result.logs : [], + ), + }; +} diff --git a/apps/api/src/browserbase/browser-automation-step-runner.service.ts b/apps/api/src/browserbase/browser-automation-step-runner.service.ts new file mode 100644 index 0000000000..daf8345f81 --- /dev/null +++ b/apps/api/src/browserbase/browser-automation-step-runner.service.ts @@ -0,0 +1,258 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { BrowserAuthProfileService } from './browser-auth-profile.service'; +import { failedBrowserEvidenceRunResult } from './browser-automation-run-result'; +import { BrowserAutomationRunStoreService } from './browser-automation-run-store.service'; +import { + profileBlockedResult, + profileMissingResult, + rollUpStepResults, + type StepForRun, +} from './browser-automation-step-results'; +import type { BrowserEvidenceLog } from './browser-evidence-execution'; +import { + createEvidenceTimeline, + type BrowserRunLivePhase, + type EvidenceTimelineStep, +} from './browser-evidence-step-timeline'; +import { + BrowserEvidenceRunnerService, + type BrowserEvidenceRunResult, +} from './browser-evidence-runner.service'; +import { BrowserbaseSessionService } from './browserbase-session.service'; + +/** " · github.com" for a step's label, or "" when the URL can't be parsed. */ +function hostSuffix(targetUrl: string): string { + try { + return ` · ${new URL(targetUrl).hostname.replace(/^www\./, '')}`; + } catch { + return ''; + } +} + +type ResolvedProfile = Awaited< + ReturnType +>; + +/** + * Runs an automation's steps in order, each on its own connection's saved + * session, and rolls the per-step results into a single run verdict. Owns the + * per-step evidence bookkeeping (BrowserAutomationStepRun) and profile health + * updates so the execution service stays focused on run lifecycle. + */ +@Injectable() +export class BrowserAutomationStepRunnerService { + private readonly logger = new Logger(BrowserAutomationStepRunnerService.name); + + constructor( + private readonly profiles: BrowserAuthProfileService = new BrowserAuthProfileService( + new BrowserbaseSessionService(), + ), + private readonly runner: BrowserEvidenceRunnerService = new BrowserEvidenceRunnerService( + new BrowserbaseSessionService(), + ), + private readonly runs: BrowserAutomationRunStoreService = new BrowserAutomationRunStoreService(), + ) {} + + /** The connection a step runs on: its bound profile, or one matched by host. */ + async resolveStepProfile(input: { + organizationId: string; + step: StepForRun; + }): Promise { + try { + if (input.step.profileId) { + const bound = await this.profiles.getProfile({ + profileId: input.step.profileId, + organizationId: input.organizationId, + }); + if (bound) return bound; + } + return await this.profiles.resolveProfileForTarget({ + organizationId: input.organizationId, + targetUrl: input.step.targetUrl, + }); + } catch { + return null; + } + } + + async runSteps(input: { + organizationId: string; + taskId: string; + automationId: string; + runId: string; + steps: StepForRun[]; + firstProfile: ResolvedProfile; + /** When set, step 0 runs on this already-open (live) session, not a new one. */ + firstSessionId?: string; + /** Live activity timeline, streamed to the Run view via realtime. */ + onSteps?: (steps: EvidenceTimelineStep[]) => void; + /** Follow each vendor's live view as the run advances to it. */ + onLiveView?: (url: string) => void; + /** Live-view phase so the UI can cover the iframe between/after vendors. */ + onLivePhase?: (phase: BrowserRunLivePhase) => void; + }): Promise { + const multiStep = input.steps.length > 1; + const timeline = createEvidenceTimeline(input.onSteps); + const results: BrowserEvidenceRunResult[] = []; + for (let index = 0; index < input.steps.length; index += 1) { + const profile = + index === 0 + ? input.firstProfile + : await this.resolveStepProfile({ + organizationId: input.organizationId, + step: input.steps[index], + }); + // Mark each vendor boundary so the combined timeline reads GH → AWS → … + if (multiStep) { + timeline.step(`Step ${index + 1}${hostSuffix(input.steps[index].targetUrl)}`); + } + results.push( + await this.runStep({ + organizationId: input.organizationId, + taskId: input.taskId, + automationId: input.automationId, + runId: input.runId, + step: input.steps[index], + index, + profile, + // Step 0 runs on the pre-opened live session (so it's watchable); + // later vendors each get their own session inside runEvidence. + sessionId: index === 0 ? input.firstSessionId : undefined, + onLog: (entry) => timeline.step(entry.message), + onLiveView: input.onLiveView, + // When this vendor's session is torn down, tell the UI whether another + // vendor is coming ("switching") or the run is wrapping up ("finishing"). + onSessionClosing: () => + input.onLivePhase?.( + index === input.steps.length - 1 ? 'finishing' : 'switching', + ), + }), + ); + } + const rolled = rollUpStepResults(results); + timeline.finish( + rolled.success ? 'done' : rolled.status === 'blocked' ? 'warn' : 'fail', + ); + return rolled; + } + + private async runStep(input: { + organizationId: string; + taskId: string; + automationId: string; + runId: string; + step: StepForRun; + index: number; + profile: ResolvedProfile; + /** Run this step on an existing (live) session instead of a fresh one. */ + sessionId?: string; + /** Per-stage progress, streamed into the run timeline. */ + onLog?: (log: BrowserEvidenceLog) => void; + /** This step's live view once its session opens (fresh-session steps only). */ + onLiveView?: (url: string) => void; + /** Fired before this step's fresh session is closed (fresh-session steps only). */ + onSessionClosing?: () => void; + }): Promise { + const stepRun = await this.runs.createStepRun({ + runId: input.runId, + stepId: input.step.id, + order: input.index, + }); + const { profile } = input; + + let result: BrowserEvidenceRunResult; + if (!profile) { + result = profileMissingResult(); + } else { + const canAutoRelogin = + profile.status === 'needs_reauth' && Boolean(profile.vaultProvider); + if (profile.status !== 'verified' && !canAutoRelogin) { + result = profileBlockedResult(profile.status); + } else { + try { + const runInput = { + organizationId: input.organizationId, + taskId: input.taskId, + automationId: input.automationId, + // Unique per step so screenshots don't collide under one run id. + runId: stepRun.id, + targetUrl: input.step.targetUrl, + instruction: input.step.instruction, + evaluationCriteria: input.step.evaluationCriteria, + profile: { + id: profile.id, + hostname: profile.hostname, + contextId: profile.contextId, + vaultProvider: profile.vaultProvider, + vaultExternalItemRef: profile.vaultExternalItemRef, + vaultConnectionId: profile.vaultConnectionId, + }, + onLog: input.onLog, + }; + result = input.sessionId + ? await this.runner.executeEvidenceOnSession({ + ...runInput, + sessionId: input.sessionId, + }) + : await this.runner.runEvidence({ + ...runInput, + onSession: input.onLiveView + ? (info) => input.onLiveView?.(info.liveViewUrl) + : undefined, + onSessionClosing: input.onSessionClosing, + }); + } catch (error) { + this.logger.error('Browser evidence runner failed', error); + result = failedBrowserEvidenceRunResult(error); + } + } + } + + await this.runs.finishStepRun({ stepRunId: stepRun.id, result }); + if (profile) { + await this.applyProfileResult({ + organizationId: input.organizationId, + profileId: profile.id, + result, + }); + } + return result; + } + + /** Reflect a run's outcome onto the connection's health (verified/reauth/blocked). */ + async applyProfileResult(input: { + organizationId: string; + profileId: string; + result: BrowserEvidenceRunResult; + }) { + if (input.result.success) { + await this.profiles.markVerified({ + organizationId: input.organizationId, + profileId: input.profileId, + }); + return; + } + + if (input.result.failureCode === 'needs_reauth') { + await this.profiles.markNeedsReauth({ + organizationId: input.organizationId, + profileId: input.profileId, + reason: input.result.blockedReason, + }); + } + + if ( + input.result.failureCode === 'captcha_blocked' || + input.result.failureCode === 'needs_user_action' + ) { + await this.profiles.markBlocked({ + organizationId: input.organizationId, + profileId: input.profileId, + reason: + input.result.blockedReason ?? + input.result.error ?? + 'User action is required before this automation can run.', + }); + } + } +} diff --git a/apps/api/src/browserbase/browser-credential-login.spec.ts b/apps/api/src/browserbase/browser-credential-login.spec.ts index 415721a857..97da611df8 100644 --- a/apps/api/src/browserbase/browser-credential-login.spec.ts +++ b/apps/api/src/browserbase/browser-credential-login.spec.ts @@ -35,7 +35,7 @@ describe('performCredentialLogin', () => { beforeEach(() => jest.useFakeTimers()); afterEach(() => jest.useRealTimers()); - it('passes secrets through act variables and never in the prompt text', async () => { + it('fills each field in its own action with secrets passed via variables', async () => { const stagehand = makeStagehand(); const password = 'sup3r-s3cret-passphrase'; @@ -47,21 +47,24 @@ describe('performCredentialLogin', () => { await jest.runAllTimersAsync(); await promise; - expect(stagehand.act).toHaveBeenNthCalledWith( - 1, - expect.stringContaining('%username%'), - { variables: { username: 'alice', password } }, - ); - expect(stagehand.act).toHaveBeenNthCalledWith( - 2, - expect.stringContaining('%code%'), - { variables: { code: '424242' } }, - ); - - // The actual secret values must not appear in the instruction sent to the LLM. - const credentialInstruction = stagehand.act.mock.calls[0][0] as string; - expect(credentialInstruction).not.toContain(password); - expect(credentialInstruction).not.toContain('alice'); + // act() does one thing per call, so username, password and the code are + // each their own step (a single combined instruction would only fill one). + const calls = stagehand.act.mock.calls as [string, unknown?][]; + const findCall = (needle: string) => + calls.find(([instruction]) => instruction.includes(needle)); + + expect(findCall('%username%')?.[1]).toEqual({ + variables: { username: 'alice' }, + }); + expect(findCall('%password%')?.[1]).toEqual({ variables: { password } }); + expect(findCall('%code%')?.[1]).toEqual({ variables: { code: '424242' } }); + + // No raw secret value may appear in any instruction sent to the LLM. + for (const [instruction] of calls) { + expect(instruction).not.toContain(password); + expect(instruction).not.toContain('alice'); + expect(instruction).not.toContain('424242'); + } }); }); @@ -106,7 +109,7 @@ describe('reloginWithStoredCredentials', () => { expect(stagehand.act).not.toHaveBeenCalled(); }); - it('signs in and returns to the target URL when verification passes', async () => { + it('signs in and verifies on the landing page without forcing a return to the URL', async () => { const stagehand = makeStagehand(); const page = makePage(); const vault: BrowserCredentialVaultAdapter = { @@ -123,11 +126,10 @@ describe('reloginWithStoredCredentials', () => { }); expect(result.isLoggedIn).toBe(true); - expect(stagehand.act).toHaveBeenCalledTimes(1); - expect(page.goto).toHaveBeenCalledWith( - baseInput.targetUrl, - expect.objectContaining({ waitUntil: 'domcontentloaded' }), - ); + expect(stagehand.act).toHaveBeenCalled(); + // We stay on the post-login landing page and verify there — navigating back + // to the entered URL can return to a login page and read as a failed sign-in. + expect(page.goto).not.toHaveBeenCalled(); }); it('retries once with a freshly resolved TOTP code', async () => { @@ -159,9 +161,8 @@ describe('reloginWithStoredCredentials', () => { }); expect(result.isLoggedIn).toBe(true); + // Resolving twice proves the second attempt used a freshly generated code. expect(resolveCredentialReference).toHaveBeenCalledTimes(2); - // Two login attempts: each enters credentials + a TOTP code (2 acts each). - expect(stagehand.act).toHaveBeenCalledTimes(4); }); it('gives up (user action required) when sign-in never authenticates', async () => { diff --git a/apps/api/src/browserbase/browser-credential-login.ts b/apps/api/src/browserbase/browser-credential-login.ts index d7f6859bda..f0470c40e3 100644 --- a/apps/api/src/browserbase/browser-credential-login.ts +++ b/apps/api/src/browserbase/browser-credential-login.ts @@ -1,15 +1,31 @@ +import { z } from 'zod'; import type { BrowserCredentialVaultAdapter, RuntimeCredentialMaterial, } from './credential-vault'; import type { BrowserbaseSessionService } from './browserbase-session.service'; -import type { BrowserEvidenceSessionInput } from './browser-evidence-runner.service'; type Stagehand = import('@browserbasehq/stagehand').Stagehand; type ActivePage = Awaited< ReturnType >; +/** + * The minimum a credential sign-in needs: which profile's stored login to + * resolve, and where to land afterward. Both the scheduler (evidence runs) and + * the connect flow's first sign-in satisfy this shape, so neither is coupled to + * the other's input type. + */ +export interface CredentialLoginTarget { + profile: { + id: string; + vaultProvider?: string | null; + vaultExternalItemRef?: string | null; + vaultConnectionId?: string | null; + }; + targetUrl: string; +} + const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); /** @@ -21,20 +37,61 @@ export async function performCredentialLogin({ stagehand, credentials, log, + usernameLabel, }: { stagehand: Stagehand; credentials: RuntimeCredentialMaterial; log: (message: string) => void; + /** The vendor's own label for the identifier field (e.g. "IAM username"), so + * the streamed step matches what's on screen. Falls back to "username". */ + usernameLabel?: string; }): Promise { - const variables: Record = {}; - if (credentials.username) variables.username = credentials.username; - if (credentials.password) variables.password = credentials.password; + // Stagehand's act() performs ONE action per call, so each field and the submit + // are separate steps — a single "enter username and password and submit" + // instruction would only fill the first field. Secrets go through `variables` + // substitution, so only placeholders (not the values) ever reach the model. + + // Site-specific fields (workspace, subdomain, …) usually precede the password. + const extraFields = credentials.extraFields ?? []; + for (const field of extraFields) { + log(`Entering ${field.label}.`); + await stagehand.act( + `Enter %value% into the "${field.label}" field on the sign-in form.`, + { variables: { value: field.value } }, + ); + await delay(1000); + } + + if (credentials.username) { + // Show the vendor's real field name in the step; the fill stays semantic so + // Stagehand still finds the field even if the label is approximate. + log(`Entering ${usernameLabel?.trim() || 'username'}.`); + await stagehand.act( + 'Enter %username% into the email or username field on the sign-in form.', + { variables: { username: credentials.username } }, + ); + await delay(1000); + // Two-step logins (Microsoft, Google, Okta) reveal the password only after a + // Next/Continue; single-page forms (e.g. GitHub) already show it. + await stagehand.act( + 'If no password field is visible yet, click the "Next" or "Continue" button to proceed. If a password field is already visible, do nothing.', + ); + await delay(1000); + } + + if (credentials.password) { + log('Entering password.'); + await stagehand.act( + 'Enter %password% into the password field on the sign-in form.', + { variables: { password: credentials.password } }, + ); + await delay(1000); + } if (credentials.username || credentials.password) { - log('Entering stored credentials.'); + log('Submitting the sign-in form.'); await stagehand.act( - 'Enter the username %username% and the password %password% into the sign-in form and submit it. If only one field is visible at a time, fill it and continue to the next step.', - { variables }, + 'Click the "Sign in" or "Log in" button to submit the sign-in form.', ); await delay(2000); } @@ -42,9 +99,13 @@ export async function performCredentialLogin({ if (credentials.totpCode) { log('Entering one-time passcode.'); await stagehand.act( - 'If a one-time passcode, two-factor authentication, or verification code field is shown, enter %code% and submit. If no such field is present, do nothing.', + 'If a one-time passcode, two-factor, or verification code field is shown, enter %code% into it. If no such field is present, do nothing.', { variables: { code: credentials.totpCode } }, ); + await delay(1500); + await stagehand.act( + 'If there is a button to submit or verify the code, click it. Otherwise do nothing.', + ); await delay(2000); } } @@ -72,7 +133,7 @@ export async function reloginWithStoredCredentials({ stagehand: Stagehand; sessions: BrowserbaseSessionService; vault: BrowserCredentialVaultAdapter; - input: BrowserEvidenceSessionInput; + input: CredentialLoginTarget; verifyLoggedIn: () => Promise; log: (message: string) => void; }): Promise { @@ -99,7 +160,6 @@ export async function reloginWithStoredCredentials({ stagehand, sessions, credentials, - input, log, }); if (await verifyLoggedIn()) return { isLoggedIn: true, page }; @@ -113,7 +173,6 @@ export async function reloginWithStoredCredentials({ stagehand, sessions, credentials: fresh, - input, log, }); if (await verifyLoggedIn()) return { isLoggedIn: true, page }; @@ -131,23 +190,123 @@ async function runLoginAttempt({ stagehand, sessions, credentials, - input, log, }: { stagehand: Stagehand; sessions: BrowserbaseSessionService; credentials: RuntimeCredentialMaterial; - input: BrowserEvidenceSessionInput; log: (message: string) => void; }): Promise { await performCredentialLogin({ stagehand, credentials, log }); - const page = await sessions.ensureActivePage(stagehand); - // Return to the target URL so the evidence step always starts from the same - // place, regardless of where the login flow redirected. - await page.goto(input.targetUrl, { - waitUntil: 'domcontentloaded', - timeoutMs: 30000, - }); + // Stay on the page the site lands on after login (typically an app/dashboard) + // and verify there. Navigating back to the entered URL can return to a login + // page — some apps always show login at the root — which reads as a failed + // sign-in even though it succeeded. The evidence instruction navigates from + // wherever we land. await delay(1500); - return page; + return sessions.ensureActivePage(stagehand); +} + +export type SignInOutcome = + | 'logged_in' + | 'invalid_credentials' + | 'needs_2fa' + | 'challenge' + | 'unknown'; + +/** + * Reads the current page after a sign-in attempt and classifies the outcome, so + * the connect flow can tell the user what happened and route them correctly. + * Never throws — an unreadable page degrades to 'unknown'. + */ +export async function classifyLoginOutcome( + stagehand: Stagehand, +): Promise { + try { + // Give the model where the browser actually is, so it can judge for itself + // whether we're on the real app or still on a sign-in / identity-provider + // page (it knows hosts like signin.aws.amazon.com or login.microsoftonline.com + // without us hardcoding URL patterns). A hint only — falls back to content. + let currentUrl = ''; + try { + const pages = stagehand.context?.pages?.() ?? []; + currentUrl = pages[pages.length - 1]?.url() ?? ''; + } catch { + // URL unavailable — classify from page content alone. + } + const { state } = await stagehand.extract( + (currentUrl ? `The browser is currently at this URL: ${currentUrl}\n\n` : '') + + 'Classify this page after a sign-in attempt, using BOTH the page content AND the URL. ' + + 'Return exactly one value: ' + + '"logged_in" — there is clear evidence the user is signed in to the actual application ' + + '(a dashboard, an account/avatar menu, or a "Sign out" control) AND the browser is on the ' + + 'application itself. If the URL is a sign-in, login, SSO, or identity-provider page, or the ' + + 'page is blank, loading, redirecting, or still shows a sign-in form or a "Sign in" / ' + + '"Log in" button, it is NOT logged_in; ' + + '"invalid_credentials" — it shows an incorrect username/email/password error; ' + + '"needs_2fa" — it asks for a two-factor, one-time, authenticator, or verification code; ' + + '"challenge" — it shows a CAPTCHA, a "verify it\'s you", a device-approval, or an email/SMS link step; ' + + '"unknown" — none of the above clearly apply.', + z.object({ + state: z.enum([ + 'logged_in', + 'invalid_credentials', + 'needs_2fa', + 'challenge', + 'unknown', + ]), + }), + ); + return state; + } catch { + return 'unknown'; + } +} + +/** + * The connect flow's first sign-in: fill the stored credentials and classify + * where we land. Unlike reloginWithStoredCredentials (used by the scheduler), + * this does NOT navigate back to the target afterward — the post-submit page is + * exactly what we need to classify (an error, a 2FA prompt, a challenge, or a + * signed-in dashboard). + */ +export async function signInAndClassify({ + stagehand, + vault, + input, + log, + usernameLabel, +}: { + stagehand: Stagehand; + vault: BrowserCredentialVaultAdapter; + input: CredentialLoginTarget; + log: (message: string) => void; + /** Vendor's identifier-field label, for a truthful streamed step. */ + usernameLabel?: string; +}): Promise<{ outcome: SignInOutcome }> { + const resolveCredentials = () => + vault.resolveCredentialReference({ + profileId: input.profile.id, + provider: input.profile.vaultProvider, + externalItemRef: input.profile.vaultExternalItemRef, + connectionId: input.profile.vaultConnectionId, + }); + + const credentials = await resolveCredentials(); + if (!credentials) return { outcome: 'unknown' }; + + await performCredentialLogin({ stagehand, credentials, log, usernameLabel }); + let outcome = await classifyLoginOutcome(stagehand); + + // A 2FA prompt still showing with a stored seed can mean the code landed on + // the ~30s rotation boundary — retry once with a freshly generated code. + if (outcome === 'needs_2fa' && credentials.totpCode) { + const fresh = await resolveCredentials(); + if (fresh?.totpCode) { + await performCredentialLogin({ stagehand, credentials: fresh, log, usernameLabel }); + outcome = await classifyLoginOutcome(stagehand); + } + } + + return { outcome }; } diff --git a/apps/api/src/browserbase/browser-credential-signin.service.spec.ts b/apps/api/src/browserbase/browser-credential-signin.service.spec.ts new file mode 100644 index 0000000000..5f03b1e5df --- /dev/null +++ b/apps/api/src/browserbase/browser-credential-signin.service.spec.ts @@ -0,0 +1,186 @@ +import { NotFoundException } from '@nestjs/common'; +import { BrowserCredentialSigninService } from './browser-credential-signin.service'; +import type { BrowserbaseSessionService } from './browserbase-session.service'; +import type { BrowserAuthProfileService } from './browser-auth-profile.service'; +import { resolveBrowserCredentialVaultAdapter } from './browser-credential-vault.factory'; + +// Avoid instantiating the real Prisma client at import time (the service pulls +// in the profile service, which imports @db). Collaborators are mocked below. +jest.mock('@db', () => ({ db: {} })); + +jest.mock('./browser-credential-vault.factory', () => ({ + resolveBrowserCredentialVaultAdapter: jest.fn(), +})); + +const mockedResolveVault = resolveBrowserCredentialVaultAdapter as jest.Mock; + +const profile = { + id: 'prof_1', + organizationId: 'org_1', + hostname: 'app.example.com', + contextId: 'ctx_1', + vaultProvider: '1password', + vaultExternalItemRef: 'op://vault/item', + vaultConnectionId: 'vault', +}; + +// `extract` classifies the page — return { state: } per call. +function makeSessions(extract: jest.Mock, act: jest.Mock) { + const page = { + goto: jest.fn().mockResolvedValue(undefined), + url: jest.fn().mockReturnValue('https://app.example.com/home'), + }; + return { + createStagehand: jest.fn().mockResolvedValue({ extract, act }), + ensureActivePage: jest.fn().mockResolvedValue(page), + safeCloseStagehand: jest.fn().mockResolvedValue(undefined), + closeSession: jest.fn().mockResolvedValue(undefined), + }; +} + +function makeProfiles(found: typeof profile | null) { + return { + getProfile: jest.fn().mockResolvedValue(found), + markVerified: jest.fn().mockResolvedValue(found), + markNeedsReauth: jest.fn().mockResolvedValue(found), + recordSignInAttempt: jest.fn().mockResolvedValue(undefined), + }; +} + +const input = { + organizationId: 'org_1', + profileId: 'prof_1', + url: 'https://app.example.com', + sessionId: 'sess_1', +}; + +const withCredentials = (creds: Record | null) => + mockedResolveVault.mockReturnValue({ + resolveCredentialReference: jest.fn().mockResolvedValue(creds), + }); + +describe('BrowserCredentialSigninService', () => { + beforeEach(() => { + jest.useFakeTimers(); + mockedResolveVault.mockReset(); + }); + afterEach(() => jest.useRealTimers()); + + const run = async ( + sessions: ReturnType, + profiles: ReturnType, + ) => { + const service = new BrowserCredentialSigninService( + sessions as unknown as BrowserbaseSessionService, + profiles as unknown as BrowserAuthProfileService, + ); + const promise = service.signInWithStoredCredentials(input); + await jest.runAllTimersAsync(); + return promise; + }; + + it('connects to the given session and never closes it', async () => { + const extract = jest.fn().mockResolvedValue({ state: 'logged_in' }); + const sessions = makeSessions(extract, jest.fn().mockResolvedValue(undefined)); + const profiles = makeProfiles(profile); + withCredentials({ username: 'u', password: 'p' }); + + await run(sessions, profiles); + + expect(sessions.createStagehand).toHaveBeenCalledWith('sess_1'); + // Release our automation handle, but leave the session open for the user. + expect(sessions.safeCloseStagehand).toHaveBeenCalledTimes(1); + expect(sessions.closeSession).not.toHaveBeenCalled(); + }); + + it('marks verified without re-entering credentials when already signed in', async () => { + const extract = jest.fn().mockResolvedValue({ state: 'logged_in' }); + const act = jest.fn().mockResolvedValue(undefined); + const sessions = makeSessions(extract, act); + const profiles = makeProfiles(profile); + withCredentials({ username: 'u', password: 'p' }); + + const result = await run(sessions, profiles); + + expect(result.isLoggedIn).toBe(true); + expect(profiles.markVerified).toHaveBeenCalledTimes(1); + expect(act).not.toHaveBeenCalled(); // already in — no navigation or fill + }); + + it('signs in with stored credentials and marks verified', async () => { + const extract = jest + .fn() + .mockResolvedValueOnce({ state: 'unknown' }) + .mockResolvedValue({ state: 'logged_in' }); + const act = jest.fn().mockResolvedValue(undefined); + const sessions = makeSessions(extract, act); + const profiles = makeProfiles(profile); + withCredentials({ username: 'user@x.com', password: 'secret' }); + + const result = await run(sessions, profiles); + + expect(result.isLoggedIn).toBe(true); + // The authenticated landing page is returned so runs can target it directly. + expect(result.homeUrl).toBe('https://app.example.com/home'); + expect(act).toHaveBeenCalled(); // navigate to sign-in + fill + expect(profiles.markVerified).toHaveBeenCalledTimes(1); + expect(profiles.markNeedsReauth).not.toHaveBeenCalled(); + // The outcome is persisted for support debugging. + expect(profiles.recordSignInAttempt).toHaveBeenCalledWith( + expect.objectContaining({ outcome: 'logged_in' }), + ); + }); + + it('reports invalid credentials as a failure and marks needs-reauth', async () => { + const extract = jest + .fn() + .mockResolvedValueOnce({ state: 'unknown' }) + .mockResolvedValue({ state: 'invalid_credentials' }); + const sessions = makeSessions(extract, jest.fn().mockResolvedValue(undefined)); + const profiles = makeProfiles(profile); + withCredentials({ username: 'user@x.com', password: 'wrong' }); + + const result = await run(sessions, profiles); + + expect(result.isLoggedIn).toBe(false); + expect(result.failure).toBe('invalid_credentials'); + expect(profiles.markNeedsReauth).toHaveBeenCalledTimes(1); + expect(profiles.markVerified).not.toHaveBeenCalled(); + // The failure reason is persisted for support debugging. + expect(profiles.recordSignInAttempt).toHaveBeenCalledWith( + expect.objectContaining({ outcome: 'invalid_credentials' }), + ); + // Session stays open so the user can take over. + expect(sessions.closeSession).not.toHaveBeenCalled(); + }); + + it('reports needs_2fa when a two-factor prompt blocks an account with no seed', async () => { + const extract = jest + .fn() + .mockResolvedValueOnce({ state: 'unknown' }) + .mockResolvedValue({ state: 'needs_2fa' }); + const sessions = makeSessions(extract, jest.fn().mockResolvedValue(undefined)); + const profiles = makeProfiles(profile); + withCredentials({ username: 'user@x.com', password: 'secret' }); // no totpCode + + const result = await run(sessions, profiles); + + expect(result.isLoggedIn).toBe(false); + expect(result.failure).toBe('needs_2fa'); + expect(profiles.markNeedsReauth).toHaveBeenCalledTimes(1); + }); + + it('throws when the profile does not exist', async () => { + const sessions = makeSessions(jest.fn(), jest.fn()); + const profiles = makeProfiles(null); + const service = new BrowserCredentialSigninService( + sessions as unknown as BrowserbaseSessionService, + profiles as unknown as BrowserAuthProfileService, + ); + + await expect( + service.signInWithStoredCredentials(input), + ).rejects.toBeInstanceOf(NotFoundException); + expect(sessions.createStagehand).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/src/browserbase/browser-credential-signin.service.ts b/apps/api/src/browserbase/browser-credential-signin.service.ts new file mode 100644 index 0000000000..4535a2d71d --- /dev/null +++ b/apps/api/src/browserbase/browser-credential-signin.service.ts @@ -0,0 +1,250 @@ +import { Injectable, Logger, NotFoundException } from '@nestjs/common'; +import { BrowserbaseSessionService } from './browserbase-session.service'; +import { BrowserAuthProfileService } from './browser-auth-profile.service'; +import { + classifyLoginOutcome, + signInAndClassify, +} from './browser-credential-login'; +import { navigateToSignIn } from './browser-login-navigation'; +import { resolveBrowserCredentialVaultAdapter } from './browser-credential-vault.factory'; + +type Stagehand = import('@browserbasehq/stagehand').Stagehand; + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +export type AutoSignInFailure = + | 'invalid_credentials' + | 'needs_2fa' + | 'challenge' + // SSO: the AI opened the identity provider; the user finishes the login there. + | 'sso_handoff' + | 'unknown'; + +export interface SignInStep { + /** Step label. */ + l: string; + /** Clock timestamp, e.g. "06:02:14". */ + t: string; + state: 'done' | 'active' | 'pending' | 'warn' | 'fail'; +} + +const clock = () => + new Date().toLocaleTimeString('en-US', { + hour12: false, + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + }); + +export interface AutoSignInResult { + isLoggedIn: boolean; + /** Why the automated sign-in couldn't complete (set only when not signed in). */ + failure?: AutoSignInFailure; + /** + * The page the app landed on after a successful login — an authenticated URL to + * target on future runs, so they reuse the session instead of re-logging in + * (e.g. sites whose root always shows a login form). + */ + homeUrl?: string; +} + +const FAILURE_REASON: Record = { + invalid_credentials: 'The stored username or password was not accepted.', + needs_2fa: 'The account requires a two-factor code to sign in.', + challenge: 'The site asked for a human verification step.', + sso_handoff: 'Finish signing in with your identity provider.', + unknown: 'Automated sign-in could not complete.', +}; + +/** + * Performs the connect flow's first sign-in on a session the caller already + * created and is showing to the user as a live view. The user watches the + * auto-fill happen; if it can't finish (wrong password, 2FA, a human + * challenge), we leave the session open on the exact page it stopped on so the + * user can take over. We only close our own Stagehand handle — never the + * session, which the caller owns. + */ +@Injectable() +export class BrowserCredentialSigninService { + private readonly logger = new Logger(BrowserCredentialSigninService.name); + + constructor( + private readonly sessions: BrowserbaseSessionService = new BrowserbaseSessionService(), + private readonly profiles: BrowserAuthProfileService = new BrowserAuthProfileService( + sessions, + ), + ) {} + + /** + * Emit the live-view URL of the tab the AI is currently on, so the connect + * flow's iframe follows it across new tabs. Best-effort — a failure just + * leaves the live view on its current tab. + */ + private async streamActiveLiveView( + stagehand: Stagehand, + sessionId: string, + onLiveView?: (url: string) => void, + ): Promise { + if (!onLiveView) return; + try { + const pages = stagehand.context?.pages?.() ?? []; + const activeUrl = pages[pages.length - 1]?.url(); + const url = await this.sessions.getActivePageLiveViewUrl(sessionId, activeUrl); + if (url) onLiveView(url); + } catch { + // Best-effort — keep the current live view. + } + } + + async signInWithStoredCredentials(input: { + organizationId: string; + profileId: string; + url: string; + sessionId: string; + /** 'password' fills stored credentials; 'sso' drives to the identity provider. */ + mode?: 'password' | 'sso'; + /** The vendor's detected identifier-field label (e.g. "IAM username"), so the + * streamed step shows the real field name instead of a generic "username". */ + usernameLabel?: string; + /** Live activity timeline, surfaced to the connect flow's activity panel. */ + onSteps?: (steps: SignInStep[]) => void; + /** Follow the AI's tab: emit the live-view URL of the page it's actually on + * (a vendor may open sign-in in a new tab) so the UI iframe can follow it. */ + onLiveView?: (url: string) => void; + }): Promise { + const mode = input.mode ?? 'password'; + const steps: SignInStep[] = []; + const emit = () => input.onSteps?.(steps.map((s) => ({ ...s }))); + // Each call advances the timeline: the prior active step becomes done and a + // new active step is appended. + const step = (label: string) => { + const last = steps[steps.length - 1]; + if (last?.state === 'active') last.state = 'done'; + steps.push({ l: label, t: clock(), state: 'active' }); + this.logger.log(`[sign-in] ${label}`); + emit(); + }; + const finish = (state: SignInStep['state']) => { + const last = steps[steps.length - 1]; + if (last) last.state = state; + emit(); + }; + // Persist the outcome of this attempt on the connection (diagnostic + // breadcrumb for support). Best-effort inside the service — never throws. + const record = (outcome: string, detail?: string | null) => + this.profiles.recordSignInAttempt({ + organizationId: input.organizationId, + profileId: input.profileId, + outcome, + detail, + }); + + const profile = await this.profiles.getProfile({ + profileId: input.profileId, + organizationId: input.organizationId, + }); + if (!profile) { + throw new NotFoundException('Browser auth profile not found'); + } + + let stagehand: Stagehand | null = null; + try { + stagehand = await this.sessions.createStagehand(input.sessionId); + const activeStagehand = stagehand; + const page = await this.sessions.ensureActivePage(activeStagehand); + step('Opening the sign-in page'); + await page.goto(input.url, { + waitUntil: 'domcontentloaded', + timeoutMs: 30000, + }); + await delay(1500); + + // The persisted context may already carry a valid session — no need to + // re-enter credentials if we're already in. The classifier is URL-aware, so + // it won't call a mid-redirect / sign-in page "logged in" (the AWS false + // positive) — it decides from the page AND where the browser actually is. + step('Checking if you’re already signed in'); + if ((await classifyLoginOutcome(activeStagehand)) === 'logged_in') { + await this.profiles.markVerified(input); + await record('logged_in', 'Existing session was still valid.'); + finish('done'); + return { isLoggedIn: true, homeUrl: page.url() }; + } + + // Get onto the actual sign-in form first — the entered URL may be a + // homepage or dashboard rather than the login page. + step('Finding the sign-in form'); + await navigateToSignIn(activeStagehand); + // The sign-in may have opened in a new tab — point the live view at + // wherever the AI actually is, so the user watches (and can complete a + // 2FA take-over on) the right page. + await this.streamActiveLiveView(activeStagehand, input.sessionId, input.onLiveView); + + // SSO: we can't hold the customer's identity-provider credentials, so the + // AI only clicks through to the provider, then hands the live browser to + // the user to finish there (their IdP login + MFA). + if (mode === 'sso') { + step('Opening your single sign-on provider'); + try { + await activeStagehand.act( + "Click the option to sign in with single sign-on (SSO) or an identity provider — buttons labeled like 'Sign in with SSO', 'Continue with Google/Microsoft/Okta', 'Use SSO', or a company login. Do not type any username or password.", + ); + } catch { + // Best effort — if we can't find the button, the user can click it in + // the live browser during take-over. + } + await delay(2500); + // The identity provider often opens in a new tab — follow it. + await this.streamActiveLiveView(activeStagehand, input.sessionId, input.onLiveView); + step('Finish signing in with your provider'); + await record('sso_handoff', FAILURE_REASON.sso_handoff); + finish('warn'); + return { isLoggedIn: false, failure: 'sso_handoff' }; + } + + const vault = resolveBrowserCredentialVaultAdapter(); + const { outcome } = await signInAndClassify({ + stagehand: activeStagehand, + vault, + input: { profile, targetUrl: input.url }, + log: step, + // On reconnect the caller doesn't pass a label (no capture form), so fall + // back to the one detected + stored at connect time. + usernameLabel: input.usernameLabel ?? profile.identifierLabel ?? undefined, + }); + step('Checking whether that worked'); + + if (outcome === 'logged_in') { + await this.profiles.markVerified(input); + await record('logged_in'); + finish('done'); + // Re-read the active page: signing in usually navigates to an app/home page. + const landed = await this.sessions.ensureActivePage(activeStagehand); + return { isLoggedIn: true, homeUrl: landed.url() }; + } + + // Narrowed to the failure states now that logged_in is handled. + await this.profiles.markNeedsReauth({ + organizationId: input.organizationId, + profileId: input.profileId, + reason: FAILURE_REASON[outcome], + }); + await record(outcome, FAILURE_REASON[outcome]); + finish('warn'); + return { isLoggedIn: false, failure: outcome }; + } catch (error) { + // An unexpected error (session/navigation/model failure) is exactly the + // kind of thing a "can't connect" ticket is about — record it, then let + // it propagate unchanged so the caller's handling is untouched. + await record( + 'error', + error instanceof Error ? error.message : 'Automated sign-in errored.', + ); + throw error; + } finally { + // Release our automation handle but leave the session open — the caller + // shows it to the user (to watch, or to take over) and closes it later. + if (stagehand) await this.sessions.safeCloseStagehand(stagehand); + } + } +} diff --git a/apps/api/src/browserbase/browser-credential-storage.service.spec.ts b/apps/api/src/browserbase/browser-credential-storage.service.spec.ts new file mode 100644 index 0000000000..91cf2b021c --- /dev/null +++ b/apps/api/src/browserbase/browser-credential-storage.service.spec.ts @@ -0,0 +1,201 @@ +import { BadRequestException } from '@nestjs/common'; +import { BrowserCredentialStorageService } from './browser-credential-storage.service'; +import * as opClient from './onepassword-client'; +import { TOTP_FIELD_TITLE, buildItemReference } from './onepassword-credential-item'; + +jest.mock('@db', () => ({ + db: { browserAuthProfile: { findFirst: jest.fn(), update: jest.fn() } }, +})); +jest.mock('./onepassword-client'); + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const { db } = require('@db'); +const findFirst = db.browserAuthProfile.findFirst as jest.Mock; +const update = db.browserAuthProfile.update as jest.Mock; + +const mockConfigured = opClient.isOnePasswordConfigured as jest.Mock; +const mockGetClient = opClient.getOnePasswordClient as jest.Mock; +const mockLoadModule = opClient.loadOnePasswordModule as jest.Mock; + +const REF = buildItemReference('vault-1', 'item-1'); + +interface Field { + id: string; + title: string; + fieldType: string; + value: string; +} + +const field = (title: string, value = 'x'): Field => ({ + id: title, + title, + fieldType: 'Text', + value, +}); + +describe('BrowserCredentialStorageService — TOTP', () => { + let service: BrowserCredentialStorageService; + let itemsGet: jest.Mock; + let itemsPut: jest.Mock; + + beforeEach(() => { + jest.clearAllMocks(); + service = new BrowserCredentialStorageService(); + mockConfigured.mockReturnValue(true); + mockLoadModule.mockResolvedValue({ ItemFieldType: { Totp: 'Totp' } }); + itemsGet = jest.fn(); + itemsPut = jest.fn().mockResolvedValue(undefined); + mockGetClient.mockResolvedValue({ items: { get: itemsGet, put: itemsPut } }); + }); + + const profile = (overrides: Record = {}) => ({ + id: 'bap_1', + organizationId: 'org_1', + vaultExternalItemRef: REF, + ...overrides, + }); + + describe('getProfileTotpStatus', () => { + it('is configured when the item has a non-empty TOTP field', async () => { + findFirst.mockResolvedValue(profile()); + itemsGet.mockResolvedValue({ + fields: [field('username'), field(TOTP_FIELD_TITLE, 'SEED')], + }); + + const result = await service.getProfileTotpStatus({ + organizationId: 'org_1', + profileId: 'bap_1', + }); + + expect(result).toEqual({ configured: true }); + }); + + it('is not configured when there is no TOTP field', async () => { + findFirst.mockResolvedValue(profile()); + itemsGet.mockResolvedValue({ fields: [field('username'), field('password')] }); + + const result = await service.getProfileTotpStatus({ + organizationId: 'org_1', + profileId: 'bap_1', + }); + + expect(result).toEqual({ configured: false }); + }); + + it('is not configured when the connection has no stored login', async () => { + findFirst.mockResolvedValue(profile({ vaultExternalItemRef: null })); + + const result = await service.getProfileTotpStatus({ + organizationId: 'org_1', + profileId: 'bap_1', + }); + + expect(result).toEqual({ configured: false }); + expect(itemsGet).not.toHaveBeenCalled(); + }); + }); + + describe('setProfileTotp', () => { + it('replaces any existing TOTP field and writes the item back', async () => { + findFirst.mockResolvedValue(profile()); + itemsGet.mockResolvedValue({ + fields: [field('username'), field(TOTP_FIELD_TITLE, 'OLD')], + }); + + const result = await service.setProfileTotp({ + organizationId: 'org_1', + profileId: 'bap_1', + totpSeed: ' NEW SEED ', + }); + + expect(result).toEqual({ configured: true }); + const written = itemsPut.mock.calls[0][0]; + const totpFields = written.fields.filter( + (f: Field) => f.title === TOTP_FIELD_TITLE, + ); + expect(totpFields).toHaveLength(1); + expect(totpFields[0]).toMatchObject({ fieldType: 'Totp', value: 'NEW SEED' }); + }); + + it('rejects when the connection has no stored login', async () => { + findFirst.mockResolvedValue(profile({ vaultExternalItemRef: null })); + + await expect( + service.setProfileTotp({ + organizationId: 'org_1', + profileId: 'bap_1', + totpSeed: 'SEED', + }), + ).rejects.toBeInstanceOf(BadRequestException); + expect(itemsPut).not.toHaveBeenCalled(); + }); + }); + + describe('storeProfileCredentials — identifier label', () => { + it('persists the detected identifier label (trimmed) on the profile', async () => { + findFirst.mockResolvedValue({ + id: 'bap_1', + organizationId: 'org_1', + displayName: 'GitHub', + hostname: 'github.com', + }); + update.mockResolvedValue({}); + mockGetClient.mockResolvedValue({ + vaults: { + list: jest.fn().mockResolvedValue([]), + create: jest.fn().mockResolvedValue({ id: 'vault-1' }), + }, + items: { create: jest.fn().mockResolvedValue({ id: 'item-1' }) }, + }); + mockLoadModule.mockResolvedValue({ + ItemCategory: { Login: 'Login' }, + ItemFieldType: { Text: 'Text', Concealed: 'Concealed', Totp: 'Totp' }, + }); + + await service.storeProfileCredentials({ + organizationId: 'org_1', + profileId: 'bap_1', + username: 'ci-bot', + password: 'secret', + usernameLabel: ' IAM username ', + }); + + expect(update).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ identifierLabel: 'IAM username' }), + }), + ); + }); + }); + + describe('clearProfileTotp', () => { + it('drops the TOTP field and writes the item back', async () => { + findFirst.mockResolvedValue(profile()); + itemsGet.mockResolvedValue({ + fields: [field('username'), field(TOTP_FIELD_TITLE, 'SEED')], + }); + + const result = await service.clearProfileTotp({ + organizationId: 'org_1', + profileId: 'bap_1', + }); + + expect(result).toEqual({ configured: false }); + const written = itemsPut.mock.calls[0][0]; + expect(written.fields.some((f: Field) => f.title === TOTP_FIELD_TITLE)).toBe(false); + }); + + it('is a no-op when there is no TOTP field', async () => { + findFirst.mockResolvedValue(profile()); + itemsGet.mockResolvedValue({ fields: [field('username')] }); + + const result = await service.clearProfileTotp({ + organizationId: 'org_1', + profileId: 'bap_1', + }); + + expect(result).toEqual({ configured: false }); + expect(itemsPut).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/apps/api/src/browserbase/browser-credential-storage.service.ts b/apps/api/src/browserbase/browser-credential-storage.service.ts index 39fd19ad66..da85ba7c46 100644 --- a/apps/api/src/browserbase/browser-credential-storage.service.ts +++ b/apps/api/src/browserbase/browser-credential-storage.service.ts @@ -1,4 +1,5 @@ import { + BadRequestException, Injectable, Logger, NotFoundException, @@ -19,6 +20,7 @@ import { USERNAME_FIELD_TITLE, buildItemReference, buildOrgVaultTitle, + parseItemReference, } from './onepassword-credential-item'; export interface StoreProfileCredentialsInput { @@ -27,6 +29,10 @@ export interface StoreProfileCredentialsInput { username: string; password: string; totpSeed?: string; + /** Extra site-specific fields (e.g. workspace, subdomain). */ + extraFields?: { label: string; value: string }[]; + /** The vendor's own label for the identifier field (e.g. "IAM username"). */ + usernameLabel?: string; } /** @@ -62,6 +68,7 @@ export class BrowserCredentialStorageService { username: input.username, password: input.password, totpSeed: input.totpSeed, + extraFields: input.extraFields, }); const { ItemCategory } = await loadOnePasswordModule(); @@ -83,10 +90,156 @@ export class BrowserCredentialStorageService { vaultProvider: ONEPASSWORD_PROVIDER, vaultExternalItemRef: itemRef, vaultConnectionId: vaultId, + // Persist the detected identifier label so reconnects and scheduled + // sign-ins show the real field name (undefined → Prisma leaves it as-is). + identifierLabel: input.usernameLabel?.trim() || undefined, }, }); } + /** + * Whether an authenticator setup key (TOTP seed) is stored for this connection + * — the source of truth for the "Automatic 2FA" status, read live from the + * vault so no DB flag can drift. Returns false (never throws) when storage + * isn't configured or the item can't be read. + */ + async getProfileTotpStatus(input: { + organizationId: string; + profileId: string; + }): Promise<{ configured: boolean }> { + if (!isOnePasswordConfigured()) return { configured: false }; + + const profile = await this.findProfile(input); + const ref = profile.vaultExternalItemRef; + if (!ref) return { configured: false }; + const { vaultId, itemId } = parseItemReference(ref); + if (!vaultId || !itemId) return { configured: false }; + + try { + const client = await getOnePasswordClient(); + const item = await client.items.get(vaultId, itemId); + const configured = item.fields.some( + (field) => field.title === TOTP_FIELD_TITLE && field.value.trim().length > 0, + ); + return { configured }; + } catch (error) { + this.logger.warn('Failed to read TOTP status from 1Password', { + error: error instanceof Error ? error.message : String(error), + }); + return { configured: false }; + } + } + + /** + * Attach or replace the authenticator setup key on a connection's existing + * login item — WITHOUT re-collecting the username/password. Requires the + * connection to already have a stored login. + */ + async setProfileTotp(input: { + organizationId: string; + profileId: string; + totpSeed: string; + }): Promise<{ configured: boolean }> { + if (!isOnePasswordConfigured()) { + throw new ServiceUnavailableException( + 'Credential storage is not configured for this environment.', + ); + } + const seed = input.totpSeed.trim(); + if (!seed) { + throw new BadRequestException('An authenticator setup key is required.'); + } + + const profile = await this.findProfile(input); + const ref = profile.vaultExternalItemRef; + const parsed = ref ? parseItemReference(ref) : { vaultId: '', itemId: '' }; + if (!parsed.vaultId || !parsed.itemId) { + throw new BadRequestException( + 'Store a login for this connection before adding an authenticator key.', + ); + } + + const client = await getOnePasswordClient(); + const { ItemFieldType } = await loadOnePasswordModule(); + const item = await client.items.get(parsed.vaultId, parsed.itemId); + item.fields = [ + ...item.fields.filter((field) => field.title !== TOTP_FIELD_TITLE), + { + id: TOTP_FIELD_TITLE, + title: TOTP_FIELD_TITLE, + fieldType: ItemFieldType.Totp, + value: seed, + }, + ]; + await client.items.put(item); + this.logger.log(`Set authenticator key for profile ${profile.id}.`); + return { configured: true }; + } + + /** + * Remove the stored authenticator setup key from a connection's login item + * ("turn off" automatic 2FA). Idempotent — a no-op when nothing is stored. + */ + async clearProfileTotp(input: { + organizationId: string; + profileId: string; + }): Promise<{ configured: boolean }> { + if (!isOnePasswordConfigured()) return { configured: false }; + + const profile = await this.findProfile(input); + const ref = profile.vaultExternalItemRef; + if (!ref) return { configured: false }; + const { vaultId, itemId } = parseItemReference(ref); + if (!vaultId || !itemId) return { configured: false }; + + const client = await getOnePasswordClient(); + const item = await client.items.get(vaultId, itemId); + const remaining = item.fields.filter( + (field) => field.title !== TOTP_FIELD_TITLE, + ); + if (remaining.length !== item.fields.length) { + item.fields = remaining; + await client.items.put(item); + this.logger.log(`Removed authenticator key for profile ${profile.id}.`); + } + return { configured: false }; + } + + private async findProfile(input: { + organizationId: string; + profileId: string; + }) { + const profile = await db.browserAuthProfile.findFirst({ + where: { id: input.profileId, organizationId: input.organizationId }, + }); + if (!profile) { + throw new NotFoundException('Browser auth profile not found'); + } + return profile; + } + + /** + * Best-effort removal of a profile's stored login from 1Password when its + * connection is deleted, so we don't leave orphaned secrets in the vault. + * Never throws — a failed cleanup shouldn't block removing the connection. + */ + async deleteProfileCredentialItem(profile: { + vaultExternalItemRef?: string | null; + }): Promise { + if (!isOnePasswordConfigured() || !profile.vaultExternalItemRef) return; + const { vaultId, itemId } = parseItemReference(profile.vaultExternalItemRef); + if (!vaultId || !itemId) return; + try { + const client = await getOnePasswordClient(); + await client.items.delete(vaultId, itemId); + this.logger.log(`Deleted 1Password item for a removed connection.`); + } catch (error) { + this.logger.warn('Failed to delete 1Password item on connection removal', { + error: error instanceof Error ? error.message : String(error), + }); + } + } + private async ensureOrgVault({ client, organizationId, @@ -113,10 +266,12 @@ export class BrowserCredentialStorageService { username, password, totpSeed, + extraFields, }: { username: string; password: string; totpSeed?: string; + extraFields?: { label: string; value: string }[]; }): Promise { const { ItemFieldType } = await loadOnePasswordModule(); const fields: ItemField[] = [ @@ -143,6 +298,17 @@ export class BrowserCredentialStorageService { }); } + for (const field of extraFields ?? []) { + const label = field.label.trim(); + if (!label || !field.value.trim()) continue; + fields.push({ + id: label, + title: label, + fieldType: ItemFieldType.Text, + value: field.value, + }); + } + return fields; } } diff --git a/apps/api/src/browserbase/browser-evidence-evaluation.ts b/apps/api/src/browserbase/browser-evidence-evaluation.ts index bd378205ba..3ac2bfba6f 100644 --- a/apps/api/src/browserbase/browser-evidence-evaluation.ts +++ b/apps/api/src/browserbase/browser-evidence-evaluation.ts @@ -16,10 +16,13 @@ export interface BrowserEvidenceEvaluator { export async function evaluateIfNeeded({ stagehand, criteria, + instruction, logs, }: { stagehand: BrowserEvidenceEvaluator; criteria?: string | null; + /** What the automation was capturing — anchors the check to the intended target. */ + instruction?: string | null; logs: BrowserEvidenceLog[]; }): Promise<{ success: boolean; @@ -43,16 +46,22 @@ export async function evaluateIfNeeded({ pass: z.boolean(), reason: z.string(), }); + const target = instruction?.trim(); const evaluation = await stagehand.extract( [ 'You are an auditor reviewing the current page after an automation has finished navigating.', - 'Decide whether the page clearly satisfies this criteria.', - 'Only return pass=true if the evidence is unambiguously present and visible.', - 'If it is ambiguous, missing, or contradicted, return pass=false.', - 'Always provide a short reason (max 220 characters).', + target + ? `The automation was asked to: "${target}". Judge the criteria about THAT specific target — ignore unrelated items that merely happen to appear on the page (e.g. a matching value belonging to a different item does not count).` + : '', + 'Decide whether the page clearly satisfies this criteria for the intended target.', + 'Only return pass=true if the evidence is unambiguously present and visible for that target.', + 'If it is ambiguous, missing, applies only to a different item, or is contradicted, return pass=false.', + 'Always provide a short reason (max 220 characters) that names the specific value and WHERE on the page it appears, so a reviewer can find it (e.g. "commit count 8,142, in the repository header").', '', `Criteria: ${normalizedCriteria}`, - ].join('\n'), + ] + .filter(Boolean) + .join('\n'), evalSchema, ); diff --git a/apps/api/src/browserbase/browser-evidence-execution.ts b/apps/api/src/browserbase/browser-evidence-execution.ts index d7fe85ad59..a46558148b 100644 --- a/apps/api/src/browserbase/browser-evidence-execution.ts +++ b/apps/api/src/browserbase/browser-evidence-execution.ts @@ -1,4 +1,5 @@ import { Logger } from '@nestjs/common'; +import sharp from 'sharp'; import { z } from 'zod'; import { renderOverlay } from './screenshot-overlay'; import type { BrowserbaseSessionService } from './browserbase-session.service'; @@ -19,7 +20,111 @@ import { reloginWithStoredCredentials } from './browser-credential-login'; type Stagehand = import('@browserbasehq/stagehand').Stagehand; -const STAGEHAND_CUA_MODEL = 'anthropic/claude-sonnet-4-6'; +// Screenshot-based navigation model. GPT-5.6 Terra is the balanced tier of +// OpenAI's newest family — strong browser-automation accuracy at roughly half +// Sol's cost — a good fit for a multi-step agent. Configurable via env for A/B +// tests (e.g. openai/gpt-5.6-sol for max accuracy), with no per-site tuning. +const DEFAULT_CUA_MODEL = 'openai/gpt-5.6-terra'; +// Claude fallback used when the primary model is unavailable (missing OpenAI key, +// preview access, rate limits, upstream errors). Must be a computer-use-capable +// model Stagehand supports — claude-sonnet-5 is NOT one; the proven Claude CUA +// options are claude-opus-4-8 / claude-sonnet-4-6 / claude-haiku-4-5. +const FALLBACK_CUA_MODEL = 'anthropic/claude-sonnet-4-6'; +// How many screenshot→action steps the agent may take. Generous so it can +// recover from a wrong turn on a complex site rather than giving up. +const CUA_MAX_STEPS = 30; + +// A `type` (not `interface`) so it stays assignable to Stagehand's model config, +// which intersects with Record. +type CuaModel = { + modelName: string; + apiKey?: string; +}; + +function resolveCuaModel(logger: Logger): CuaModel { + const requested = process.env.BROWSERBASE_CUA_MODEL || DEFAULT_CUA_MODEL; + if (requested.startsWith('openai/') && !process.env.OPENAI_API_KEY) { + logger.warn( + `OPENAI_API_KEY not set; falling back from ${requested} to ${FALLBACK_CUA_MODEL} for navigation.`, + ); + return { + modelName: FALLBACK_CUA_MODEL, + apiKey: process.env.ANTHROPIC_API_KEY, + }; + } + return { + modelName: requested, + apiKey: requested.startsWith('openai/') + ? process.env.OPENAI_API_KEY + : process.env.ANTHROPIC_API_KEY, + }; +} + +function claudeFallbackModel(): CuaModel { + return { modelName: FALLBACK_CUA_MODEL, apiKey: process.env.ANTHROPIC_API_KEY }; +} + +async function runCuaNavigation({ + stagehand, + instruction, + model, +}: { + stagehand: Stagehand; + instruction: string; + model: CuaModel; +}): Promise { + await stagehand + .agent({ mode: 'cua', model }) + .execute({ instruction, maxSteps: CUA_MAX_STEPS }); +} + +/** + * The agent's `.execute()` is a single opaque call, but it opens/switches tabs + * mid-run (e.g. a sign-in or console tab). Poll the newest tab's URL and, when it + * changes, push its live-view URL so a watched run follows the active tab instead + * of showing the stale initial one. Best-effort: never lets live-view following + * disturb the run. Returns a stop function. + */ +function startLiveViewFollower({ + stagehand, + sessions, + sessionId, + onLiveView, +}: { + stagehand: Stagehand; + sessions: BrowserbaseSessionService; + sessionId: string; + onLiveView: (url: string) => void; +}): () => void { + let lastUrl = ''; + let stopped = false; + let inFlight = false; + + const tick = async () => { + if (stopped || inFlight) return; + inFlight = true; + try { + const pages = stagehand.context?.pages?.() ?? []; + const url = pages.at(-1)?.url?.() ?? ''; + if (url && url !== lastUrl) { + lastUrl = url; + const liveUrl = await sessions.getActivePageLiveViewUrl(sessionId, url); + if (liveUrl && !stopped) onLiveView(liveUrl); + } + } catch { + // Best-effort — following the tab must never affect the run. + } finally { + inFlight = false; + } + }; + + const interval = setInterval(() => void tick(), 2500); + return () => { + stopped = true; + clearInterval(interval); + }; +} + const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); export interface BrowserEvidenceLog { @@ -31,6 +136,8 @@ export interface BrowserEvidenceLog { export interface BrowserEvidenceExecutionResult { success: boolean; screenshot?: string; + /** A close-up: the agent's final viewport (where it left the evidence). */ + focusScreenshot?: string; finalUrl?: string; evaluationStatus?: 'pass' | 'fail'; evaluationReason?: string; @@ -47,15 +154,27 @@ export async function executeBrowserEvidence({ sessions, logger, vault, + onLog, + onLiveView, }: { input: BrowserEvidenceSessionInput; sessions: BrowserbaseSessionService; logger: Logger; vault: BrowserCredentialVaultAdapter; + /** Called as each stage begins, so a live test run can stream its progress. */ + onLog?: (log: BrowserEvidenceLog) => void; + /** Called with the active tab's live-view URL so a watched run follows tabs. */ + onLiveView?: (url: string) => void; }): Promise { const logs: BrowserEvidenceLog[] = []; const log = (stage: string, message: string) => { - logs.push({ timestamp: new Date().toISOString(), stage, message }); + const entry: BrowserEvidenceLog = { + timestamp: new Date().toISOString(), + stage, + message, + }; + logs.push(entry); + onLog?.(entry); }; let stagehand: Stagehand | null = null; let currentStage: BrowserAutomationFailureStage = 'session'; @@ -77,13 +196,22 @@ export async function executeBrowserEvidence({ }); await delay(1000); + // Point a watched run's live view at the current tab up front, so it's + // correct before the agent starts opening/switching tabs. + if (onLiveView) { + const initialLiveView = await sessions + .getActivePageLiveViewUrl(input.sessionId, page.url()) + .catch(() => null); + if (initialLiveView) onLiveView(initialLiveView); + } + currentStage = 'auth'; const authCheck = await checkAuth(stagehand); if (!authCheck.isLoggedIn) { log( 'auth', - 'Session expired; attempting sign-in with stored credentials.', + 'Not signed in on this page — signing in with stored credentials.', ); const relogin = await reloginWithStoredCredentials({ stagehand: activeStagehand, @@ -117,16 +245,46 @@ export async function executeBrowserEvidence({ currentStage = 'action'; log('action', 'Running navigation instruction.'); - const instruction = `${input.instruction}. After completing all navigation steps, stop and wait.`; - await stagehand - .agent({ - cua: true, - model: { - modelName: STAGEHAND_CUA_MODEL, - apiKey: process.env.ANTHROPIC_API_KEY, - }, - }) - .execute({ instruction, maxSteps: 20 }); + // Find its own way (no exact directions needed), self-correct a wrong turn, + // and read what's already on screen instead of over-navigating. + const instruction = `${input.instruction}. Work out the path yourself — you don't need exact directions. If the instruction names a specific item or page, open it so the final screenshot clearly shows just that item, not a long list of many. Before finishing, verify the page matches what was asked and correct it if you opened the wrong thing. Don't take unnecessary detours. When the right thing is clearly shown, stop and wait.`; + const primaryModel = resolveCuaModel(logger); + // Follow the agent across tabs while it works (test/watched runs only). + const stopFollowing = onLiveView + ? startLiveViewFollower({ + stagehand: activeStagehand, + sessions, + sessionId: input.sessionId, + onLiveView, + }) + : () => {}; + try { + try { + await runCuaNavigation({ + stagehand: activeStagehand, + instruction, + model: primaryModel, + }); + } catch (navError) { + // The navigation model can be unavailable at runtime (preview access, rate + // limits, upstream errors). Fall back to Claude once — unless we were + // already on it — rather than failing the whole run. + if (primaryModel.modelName === FALLBACK_CUA_MODEL) throw navError; + logger.warn( + `Navigation with ${primaryModel.modelName} failed; retrying with ${FALLBACK_CUA_MODEL}. ${ + navError instanceof Error ? navError.message : String(navError) + }`, + ); + log('action', 'Primary navigation model unavailable — retrying with a backup model.'); + await runCuaNavigation({ + stagehand: activeStagehand, + instruction, + model: claudeFallbackModel(), + }); + } + } finally { + stopFollowing(); + } await delay(2000); page = await resolveEvidencePage({ @@ -138,7 +296,18 @@ export async function executeBrowserEvidence({ currentStage = 'screenshot'; log('screenshot', 'Capturing screenshot.'); + // Full page, not the viewport: a short page would otherwise leave a tall + // band of empty viewport below the content (a wide gap above our overlay), + // and the full page is more complete evidence. const rawScreenshot = await page.screenshot({ + type: 'jpeg', + quality: 80, + fullPage: true, + }); + // The current viewport — the agent stops with the evidence on screen, so this + // is a naturally focused close-up (one screen, no scroll noise) to sit + // alongside the full-page shot as evidence. + const rawFocus = await page.screenshot({ type: 'jpeg', quality: 80, fullPage: false, @@ -151,11 +320,33 @@ export async function executeBrowserEvidence({ instruction: input.instruction, finalUrl, }); + // Only keep the close-up when it actually differs — i.e. the page is taller + // than one screen. If the page fits in the viewport, the two shots are the + // same, so we'd just show a duplicate. + let focusScreenshot: string | undefined; + try { + const [fullMeta, focusMeta] = await Promise.all([ + sharp(rawScreenshot).metadata(), + sharp(rawFocus).metadata(), + ]); + if ((fullMeta.height ?? 0) > (focusMeta.height ?? 0) * 1.15) { + focusScreenshot = await renderScreenshot({ + logger, + logs, + rawScreenshot: rawFocus, + instruction: input.instruction, + finalUrl, + }); + } + } catch { + // If we can't measure, skip the close-up rather than risk a duplicate. + } currentStage = 'evaluation'; await bringEvidencePageToFront(page); const evaluation = await evaluateIfNeeded({ stagehand, criteria: input.evaluationCriteria, + instruction: input.instruction, logs, }); @@ -163,6 +354,7 @@ export async function executeBrowserEvidence({ return { success: false, screenshot, + focusScreenshot, finalUrl, evaluationStatus: evaluation.evaluationStatus, evaluationReason: evaluation.evaluationReason, @@ -176,6 +368,7 @@ export async function executeBrowserEvidence({ return { success: true, screenshot, + focusScreenshot, finalUrl, evaluationStatus: evaluation.evaluationStatus, evaluationReason: evaluation.evaluationReason, @@ -232,8 +425,12 @@ async function checkAuth( stagehand: Stagehand, ): Promise<{ isLoggedIn: boolean }> { const loginSchema = z.object({ isLoggedIn: z.boolean() }); + // Detect the sign-in state by the presence of a login prompt rather than by + // an avatar/profile menu: many apps have no obvious "logged-in" marker, so + // requiring one produced false negatives (and needless re-logins). Treat the + // page as logged in unless a sign-in prompt is clearly the ask. return stagehand.extract( - 'Check if the user is logged in to this website. Look for a user avatar, profile menu, account dropdown, or login/sign-in buttons. Return true if logged in, false if you see login buttons or a login form.', + 'Is this page asking the user to sign in? Look for a visible login prompt: a password field, username/email + password fields, an SSO/"Continue with" screen, or a page whose main call to action is Sign in / Log in. Return isLoggedIn=false ONLY if such a sign-in prompt is clearly present. For any ordinary application or content page with no sign-in prompt, return isLoggedIn=true.', loginSchema, ); } diff --git a/apps/api/src/browserbase/browser-evidence-page.spec.ts b/apps/api/src/browserbase/browser-evidence-page.spec.ts index df155bfeaa..4692ae1322 100644 --- a/apps/api/src/browserbase/browser-evidence-page.spec.ts +++ b/apps/api/src/browserbase/browser-evidence-page.spec.ts @@ -43,6 +43,41 @@ describe('selectEvidencePage', () => { expect(selected).toBe(newTab); }); + it('follows a new tab the agent opened on a different host (AWS console vs sign-in)', () => { + const initialPage = page({ id: 'home', url: 'https://aws.amazon.com/' }); + const iamTab = page({ + id: 'iam', + url: 'https://console.aws.amazon.com/iam/home#/users', + }); + + const selected = selectEvidencePage({ + pages: [initialPage, iamTab], + initialPage, + // Entered host matches neither tab — the screenshot must still land on the + // page the agent navigated to, not the stale homepage. + targetUrl: 'https://us-east-2.signin.aws.amazon.com/', + isClosed, + }); + + expect(selected).toBe(iamTab); + }); + + it('keeps the initial page when the agent navigated within it (no new tab)', () => { + const initialPage = page({ + id: 'initial', + url: 'https://console.aws.amazon.com/iam/home', + }); + + const selected = selectEvidencePage({ + pages: [initialPage], + initialPage, + targetUrl: 'https://us-east-2.signin.aws.amazon.com/', + isClosed, + }); + + expect(selected).toBe(initialPage); + }); + it('returns null when every page is closed', () => { const initialPage = page({ closed: true, diff --git a/apps/api/src/browserbase/browser-evidence-page.ts b/apps/api/src/browserbase/browser-evidence-page.ts index ee12e64d9f..28869f14d8 100644 --- a/apps/api/src/browserbase/browser-evidence-page.ts +++ b/apps/api/src/browserbase/browser-evidence-page.ts @@ -50,15 +50,22 @@ export function selectEvidencePage({ const openMatchesTarget = (page: Page) => targetHostname !== null && safeHostname(page.url()) === targetHostname; - const newestMatchingNewPage = [...openPages] - .reverse() - .find((page) => page !== initialPage && openMatchesTarget(page)); + const reversed = [...openPages].reverse(); + + // A new tab that matches the entered host (a popup straight to the target). + const newestMatchingNewPage = reversed.find( + (page) => page !== initialPage && openMatchesTarget(page), + ); if (newestMatchingNewPage) return newestMatchingNewPage; - if (!isClosed(initialPage)) return initialPage; + // The instruction often navigates to a DIFFERENT host than the entered URL + // (e.g. AWS console vs the sign-in host), commonly in a new tab. Prefer the + // newest non-initial page over the (now stale) initial one, so the screenshot + // matches where the agent actually ended up — not the starting/homepage tab. + const newestNonInitial = reversed.find((page) => page !== initialPage); + if (newestNonInitial) return newestNonInitial; - const newestMatchingPage = [...openPages].reverse().find(openMatchesTarget); - if (newestMatchingPage) return newestMatchingPage; + if (!isClosed(initialPage)) return initialPage; return openPages.at(-1) ?? null; } diff --git a/apps/api/src/browserbase/browser-evidence-runner.service.spec.ts b/apps/api/src/browserbase/browser-evidence-runner.service.spec.ts index 1f615889d4..699476c5ef 100644 --- a/apps/api/src/browserbase/browser-evidence-runner.service.spec.ts +++ b/apps/api/src/browserbase/browser-evidence-runner.service.spec.ts @@ -98,4 +98,45 @@ describe('BrowserEvidenceRunnerService', () => { const call = jest.mocked(executeBrowserEvidence).mock.calls[0]; expect(call?.[0].input).not.toHaveProperty('credentialMaterial'); }); + + it('signals the imminent teardown before closing a fresh session', async () => { + const sessions = new BrowserbaseSessionService(); + jest.spyOn(sessions, 'createSessionWithContext').mockResolvedValue({ + sessionId: 'sess_new', + liveViewUrl: 'https://live.example/sess_new', + }); + const closeSpy = jest + .spyOn(sessions, 'closeSession') + .mockResolvedValue(undefined); + + jest.mocked(executeBrowserEvidence).mockResolvedValue({ + success: true, + finalUrl: 'https://example.com/final', + logs: [], + }); + + const service = new BrowserEvidenceRunnerService( + sessions, + new BrowserbaseScreenshotService(), + ); + + const onSessionClosing = jest.fn(); + await service.runEvidence({ + organizationId: 'org_1', + automationId: 'bau_1', + runId: 'bar_1', + targetUrl: 'https://example.com', + instruction: 'collect evidence', + profile: { id: 'bap_1', hostname: 'example.com', contextId: 'ctx_1' }, + onSessionClosing, + }); + + expect(onSessionClosing).toHaveBeenCalledTimes(1); + expect(closeSpy).toHaveBeenCalledWith('sess_new'); + // Must fire BEFORE the close, so the UI can cover the iframe before the live + // session's socket drops (the whole point of the transition overlay). + expect(onSessionClosing.mock.invocationCallOrder[0]).toBeLessThan( + closeSpy.mock.invocationCallOrder[0], + ); + }); }); diff --git a/apps/api/src/browserbase/browser-evidence-runner.service.ts b/apps/api/src/browserbase/browser-evidence-runner.service.ts index 84aa1b8487..ff317e29cc 100644 --- a/apps/api/src/browserbase/browser-evidence-runner.service.ts +++ b/apps/api/src/browserbase/browser-evidence-runner.service.ts @@ -35,6 +35,17 @@ export interface BrowserEvidenceRunnerInput { vaultConnectionId?: string | null; }; beforeExecution?: () => Promise; + /** Live per-stage progress callback (used to stream a test run's activity). */ + onLog?: (log: BrowserEvidenceLog) => void; + /** Fired as the agent switches tabs, so a watched run's live view follows it. */ + onLiveView?: (url: string) => void; + /** Fired once this step's live session opens, so the Run view can follow it. */ + onSession?: (info: { sessionId: string; liveViewUrl: string }) => void; + /** + * Fired right before this step's session is torn down, so the Run view can + * cover the (about-to-disconnect) live iframe with a transition state. + */ + onSessionClosing?: () => void; } export interface BrowserEvidenceSessionInput extends BrowserEvidenceRunnerInput { @@ -46,6 +57,9 @@ export interface BrowserEvidenceRunResult { status: 'completed' | 'failed' | 'blocked'; screenshotKey?: string; screenshotUrl?: string; + /** A focused close-up (the agent's final viewport) shown beside the full page. */ + focusScreenshotKey?: string; + focusScreenshotUrl?: string; finalUrl?: string; evaluationStatus?: 'pass' | 'fail'; evaluationReason?: string; @@ -84,9 +98,10 @@ export class BrowserEvidenceRunnerService { profileId: input.profile.id, hostname: input.profile.hostname, run: async () => { - const { sessionId } = await this.sessions.createSessionWithContext( - input.profile.contextId, - ); + const { sessionId, liveViewUrl } = + await this.sessions.createSessionWithContext(input.profile.contextId); + // Surface this step's live view so a watched run can follow each vendor. + input.onSession?.({ sessionId, liveViewUrl }); try { return await this.executeEvidenceOnSessionUnlocked({ @@ -94,6 +109,9 @@ export class BrowserEvidenceRunnerService { sessionId, }); } finally { + // Signal the imminent teardown before we actually close, so the UI can + // cover the live view before Browserbase's iframe shows "disconnected". + input.onSessionClosing?.(); await this.closeSession(sessionId); } }, @@ -120,9 +138,15 @@ export class BrowserEvidenceRunnerService { sessions: this.sessions, logger: this.logger, vault: this.vault, + onLog: input.onLog, + onLiveView: input.onLiveView, }); - let uploaded: { screenshotKey: string; screenshotUrl: string } | null = - null; + let uploaded: { + screenshotKey?: string; + screenshotUrl?: string; + focusScreenshotKey?: string; + focusScreenshotUrl?: string; + } | null = null; try { uploaded = await this.uploadCapturedScreenshot({ input, execution }); } catch (err) { @@ -146,6 +170,8 @@ export class BrowserEvidenceRunnerService { status: this.blockedStatusForCode(execution.failureCode), screenshotKey: uploaded?.screenshotKey, screenshotUrl: uploaded?.screenshotUrl, + focusScreenshotKey: uploaded?.focusScreenshotKey, + focusScreenshotUrl: uploaded?.focusScreenshotUrl, finalUrl: execution.finalUrl, evaluationStatus: execution.evaluationStatus, evaluationReason: execution.evaluationReason, @@ -163,6 +189,8 @@ export class BrowserEvidenceRunnerService { status: 'completed', screenshotKey: uploaded?.screenshotKey, screenshotUrl: uploaded?.screenshotUrl, + focusScreenshotKey: uploaded?.focusScreenshotKey, + focusScreenshotUrl: uploaded?.focusScreenshotUrl, finalUrl: execution.finalUrl, evaluationStatus: execution.evaluationStatus, evaluationReason: execution.evaluationReason, @@ -170,26 +198,50 @@ export class BrowserEvidenceRunnerService { }; } + private async uploadOne( + input: BrowserEvidenceRunnerInput, + base64: string, + variant?: string, + ): Promise<{ key: string; url: string }> { + // A variant keys to a distinct object (…/runId-focus.jpg) so it doesn't + // overwrite the full-page shot. + const key = await this.screenshots.uploadScreenshot({ + organizationId: input.organizationId, + automationId: input.automationId, + runId: variant ? `${input.runId}-${variant}` : input.runId, + base64Screenshot: base64, + }); + const url = await this.screenshots.getPresignedUrl({ key }); + return { key, url }; + } + private async uploadCapturedScreenshot({ input, execution, }: { input: BrowserEvidenceRunnerInput; execution: BrowserEvidenceExecutionResult; - }): Promise<{ screenshotKey: string; screenshotUrl: string } | null> { + }): Promise<{ + screenshotKey?: string; + screenshotUrl?: string; + focusScreenshotKey?: string; + focusScreenshotUrl?: string; + } | null> { if (!execution.screenshot) return null; - const screenshotKey = await this.screenshots.uploadScreenshot({ - organizationId: input.organizationId, - automationId: input.automationId, - runId: input.runId, - base64Screenshot: execution.screenshot, - }); - const screenshotUrl = await this.screenshots.getPresignedUrl({ - key: screenshotKey, - }); + const [full, focus] = await Promise.all([ + this.uploadOne(input, execution.screenshot), + execution.focusScreenshot + ? this.uploadOne(input, execution.focusScreenshot, 'focus') + : Promise.resolve(null), + ]); - return { screenshotKey, screenshotUrl }; + return { + screenshotKey: full.key, + screenshotUrl: full.url, + focusScreenshotKey: focus?.key, + focusScreenshotUrl: focus?.url, + }; } private async closeSession(sessionId: string): Promise { diff --git a/apps/api/src/browserbase/browser-evidence-step-timeline.ts b/apps/api/src/browserbase/browser-evidence-step-timeline.ts new file mode 100644 index 0000000000..a33415807c --- /dev/null +++ b/apps/api/src/browserbase/browser-evidence-step-timeline.ts @@ -0,0 +1,60 @@ +/** + * A live activity timeline for an evidence run — the same "steps" the composer's + * Test panel and the connect/sign-in flow show. Each engine log advances the + * timeline (the prior active step becomes done, a new active step is appended), + * and the whole snapshot is emitted so a Trigger.dev task can publish it to + * realtime metadata for the UI to render. + */ +export interface EvidenceTimelineStep { + /** Step label. */ + l: string; + /** Clock timestamp, e.g. "06:02:14". */ + t: string; + state: 'done' | 'active' | 'pending' | 'warn' | 'fail'; +} + +/** + * What the Run view's live iframe should reflect. Between vendors and at the end + * of a run a browser session is torn down, and the (now-dead) Browserbase live + * view briefly shows its own "disconnected" notice. We stream this phase so the + * UI can cover the iframe with a calm transition state instead: + * • running — a live session is up; show the iframe + * • switching — the current vendor is done; the next vendor's session is opening + * • finishing — the last vendor is done; the run is wrapping up + */ +export type BrowserRunLivePhase = 'running' | 'switching' | 'finishing'; + +const clock = () => + new Date().toLocaleTimeString('en-US', { + hour12: false, + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + }); + +export interface EvidenceTimeline { + /** Advance the timeline with a new active step (marks the prior one done). */ + step: (label: string) => void; + /** Set the final state of the last step (done/warn/fail). */ + finish: (state: EvidenceTimelineStep['state']) => void; +} + +export function createEvidenceTimeline( + onSteps?: (steps: EvidenceTimelineStep[]) => void, +): EvidenceTimeline { + const steps: EvidenceTimelineStep[] = []; + const emit = () => onSteps?.(steps.map((s) => ({ ...s }))); + return { + step(label: string) { + const last = steps[steps.length - 1]; + if (last?.state === 'active') last.state = 'done'; + steps.push({ l: label, t: clock(), state: 'active' }); + emit(); + }, + finish(state: EvidenceTimelineStep['state']) { + const last = steps[steps.length - 1]; + if (last) last.state = state; + emit(); + }, + }; +} diff --git a/apps/api/src/browserbase/browser-instruction-test.service.spec.ts b/apps/api/src/browserbase/browser-instruction-test.service.spec.ts new file mode 100644 index 0000000000..079411b429 --- /dev/null +++ b/apps/api/src/browserbase/browser-instruction-test.service.spec.ts @@ -0,0 +1,160 @@ +import { BrowserInstructionTestService } from './browser-instruction-test.service'; +import type { BrowserbaseSessionService } from './browserbase-session.service'; +import type { BrowserAuthProfileService } from './browser-auth-profile.service'; +import type { BrowserEvidenceRunnerService } from './browser-evidence-runner.service'; +import type { + BrowserEvidenceRunResult, + BrowserEvidenceSessionInput, +} from './browser-evidence-runner.service'; + +// The service imports the profile/runner services which pull in @db; mock it so +// the real Prisma client isn't instantiated at import time. +jest.mock('@db', () => ({ db: {} })); + +const profile = { + id: 'prof_1', + hostname: 'app.example.com', + contextId: 'ctx_1', + vaultProvider: '1password', + vaultExternalItemRef: 'op://vault/item', + vaultConnectionId: 'vault', +}; + +function makeProfiles() { + return { + resolveProfileForTarget: jest.fn().mockResolvedValue(profile), + }; +} + +/** A runner whose `executeEvidenceOnSession` streams a couple of stages, then returns `result`. */ +function makeRunner( + result: Partial & { success: boolean; status: BrowserEvidenceRunResult['status'] }, +) { + return { + executeEvidenceOnSession: jest.fn( + async (input: BrowserEvidenceSessionInput) => { + input.onLog?.({ timestamp: 't1', stage: 'navigation', message: 'Opening the page' }); + input.onLog?.({ timestamp: 't2', stage: 'action', message: 'Running instruction' }); + return { logs: [], ...result } as BrowserEvidenceRunResult; + }, + ), + }; +} + +const baseInput = { + organizationId: 'org_1', + taskId: 'task_1', + profileId: 'prof_1', + targetUrl: 'https://app.example.com/settings', + instruction: 'Screenshot the MFA policy', + evaluationCriteria: 'MFA is enforced', + sessionId: 'sess_1', +}; + +function build(runner: ReturnType, profiles = makeProfiles()) { + const service = new BrowserInstructionTestService( + {} as unknown as BrowserbaseSessionService, + profiles as unknown as BrowserAuthProfileService, + runner as unknown as BrowserEvidenceRunnerService, + ); + return { service, profiles }; +} + +describe('BrowserInstructionTestService', () => { + it('runs the ad-hoc instruction through the evidence runner with synthetic ids', async () => { + const runner = makeRunner({ success: true, status: 'completed', screenshotUrl: 'https://s3/x.png' }); + const { service, profiles } = build(runner); + + await service.testInstructionOnSession(baseInput); + + expect(profiles.resolveProfileForTarget).toHaveBeenCalledWith({ + organizationId: 'org_1', + targetUrl: 'https://app.example.com/settings', + profileId: 'prof_1', + }); + const call = runner.executeEvidenceOnSession.mock.calls[0][0]; + expect(call.instruction).toBe('Screenshot the MFA policy'); + expect(call.evaluationCriteria).toBe('MFA is enforced'); + expect(call.sessionId).toBe('sess_1'); + expect(call.automationId).toContain('test-'); + expect(call.runId).toContain('test-'); + expect(call.profile.contextId).toBe('ctx_1'); + }); + + it('streams accumulating steps and marks the last one done on success', async () => { + const runner = makeRunner({ success: true, status: 'completed', screenshotUrl: 'https://s3/x.png' }); + const { service } = build(runner); + const frames: { l: string; state: string }[][] = []; + + const result = await service.testInstructionOnSession({ + ...baseInput, + onSteps: (steps) => frames.push(steps.map((s) => ({ l: s.l, state: s.state }))), + }); + + // First stage active, second stage flips the first to done and appends active. + expect(frames[0]).toEqual([{ l: 'Opening the page', state: 'active' }]); + expect(frames[1]).toEqual([ + { l: 'Opening the page', state: 'done' }, + { l: 'Running instruction', state: 'active' }, + ]); + // Final frame: last step resolved to done. + const last = frames[frames.length - 1]; + expect(last[last.length - 1]).toEqual({ l: 'Running instruction', state: 'done' }); + expect(result.success).toBe(true); + expect(result.screenshotUrl).toBe('https://s3/x.png'); + }); + + it('maps the runner verdict into the test result', async () => { + const runner = makeRunner({ + success: true, + status: 'completed', + evaluationStatus: 'fail', + evaluationReason: 'MFA optional', + screenshotUrl: 'https://s3/x.png', + }); + const { service } = build(runner); + + const result = await service.testInstructionOnSession(baseInput); + + expect(result.evaluationStatus).toBe('fail'); + expect(result.evaluationReason).toBe('MFA optional'); + }); + + it('marks the last step failed when the run fails', async () => { + const runner = makeRunner({ + success: false, + status: 'failed', + error: 'stuck', + failureCode: 'timeout', + }); + const { service } = build(runner); + let final: { state: string }[] = []; + + const result = await service.testInstructionOnSession({ + ...baseInput, + onSteps: (steps) => (final = steps.map((s) => ({ state: s.state }))), + }); + + expect(final[final.length - 1].state).toBe('fail'); + expect(result.success).toBe(false); + expect(result.error).toBe('stuck'); + }); + + it('marks the last step as a warning when the connection needs reauth', async () => { + const runner = makeRunner({ + success: false, + status: 'blocked', + needsReauth: true, + error: 'session expired', + }); + const { service } = build(runner); + let final: { state: string }[] = []; + + await service.testInstructionOnSession({ + ...baseInput, + onSteps: (steps) => (final = steps.map((s) => ({ state: s.state }))), + }); + + expect(final[final.length - 1].state).toBe('warn'); + }); +}); diff --git a/apps/api/src/browserbase/browser-instruction-test.service.ts b/apps/api/src/browserbase/browser-instruction-test.service.ts new file mode 100644 index 0000000000..78abe7b5fa --- /dev/null +++ b/apps/api/src/browserbase/browser-instruction-test.service.ts @@ -0,0 +1,116 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { BrowserbaseSessionService } from './browserbase-session.service'; +import { BrowserAuthProfileService } from './browser-auth-profile.service'; +import { BrowserEvidenceRunnerService } from './browser-evidence-runner.service'; +import type { BrowserEvidenceLog } from './browser-evidence-execution'; +import { + createEvidenceTimeline, + type EvidenceTimelineStep, +} from './browser-evidence-step-timeline'; + +/** A single line in the live test-run activity timeline (mirrors the sign-in flow). */ +export type InstructionTestStep = EvidenceTimelineStep; + +export interface InstructionTestResult { + success: boolean; + screenshotUrl?: string; + /** A focused close-up (the agent's final viewport) shown beside the full page. */ + focusScreenshotUrl?: string; + finalUrl?: string; + evaluationStatus?: 'pass' | 'fail'; + evaluationReason?: string; + error?: string; + needsReauth?: boolean; + failureCode?: string; + blockedReason?: string; +} + +/** + * Runs an instruction the user hasn't saved yet against the connection's live + * session, so they can watch it work before committing it to the schedule. + * + * It reuses the exact evidence runner scheduled runs use, so a passing test + * means the saved instruction will behave the same unattended. Nothing is + * persisted — no automation, no run record — this only proves the instruction + * out. The caller owns the session (it shows the live view); we never close it. + */ +@Injectable() +export class BrowserInstructionTestService { + private readonly logger = new Logger(BrowserInstructionTestService.name); + + constructor( + private readonly sessions: BrowserbaseSessionService = new BrowserbaseSessionService(), + private readonly profiles: BrowserAuthProfileService = new BrowserAuthProfileService( + sessions, + ), + private readonly runner: BrowserEvidenceRunnerService = new BrowserEvidenceRunnerService( + sessions, + ), + ) {} + + async testInstructionOnSession(input: { + organizationId: string; + taskId?: string; + profileId?: string; + targetUrl: string; + instruction: string; + evaluationCriteria?: string; + sessionId: string; + /** Live activity timeline, surfaced to the composer's test panel. */ + onSteps?: (steps: InstructionTestStep[]) => void; + /** Active-tab live-view URL, so the test panel follows the agent across tabs. */ + onLiveView?: (url: string) => void; + }): Promise { + const timeline = createEvidenceTimeline(input.onSteps); + + const profile = await this.profiles.resolveProfileForTarget({ + organizationId: input.organizationId, + targetUrl: input.targetUrl, + profileId: input.profileId, + }); + + const result = await this.runner.executeEvidenceOnSession({ + organizationId: input.organizationId, + taskId: input.taskId, + // Synthetic ids: nothing is persisted, but the runner uses them for the + // screenshot key path and its own logging. + automationId: `test-${profile.id}`, + runId: `test-${input.sessionId}`, + sessionId: input.sessionId, + targetUrl: input.targetUrl, + instruction: input.instruction, + evaluationCriteria: input.evaluationCriteria, + profile: { + id: profile.id, + hostname: profile.hostname, + contextId: profile.contextId, + vaultProvider: profile.vaultProvider, + vaultExternalItemRef: profile.vaultExternalItemRef, + vaultConnectionId: profile.vaultConnectionId, + }, + onLog: (entry: BrowserEvidenceLog) => timeline.step(entry.message), + onLiveView: input.onLiveView, + }); + + timeline.finish( + result.success + ? 'done' + : result.status === 'blocked' || result.needsReauth + ? 'warn' + : 'fail', + ); + + return { + success: result.success, + screenshotUrl: result.screenshotUrl, + focusScreenshotUrl: result.focusScreenshotUrl, + finalUrl: result.finalUrl, + evaluationStatus: result.evaluationStatus, + evaluationReason: result.evaluationReason, + error: result.error, + needsReauth: result.needsReauth, + failureCode: result.failureCode, + blockedReason: result.blockedReason, + }; + } +} diff --git a/apps/api/src/browserbase/browser-login-analysis.spec.ts b/apps/api/src/browserbase/browser-login-analysis.spec.ts new file mode 100644 index 0000000000..ff31f7c4a1 --- /dev/null +++ b/apps/api/src/browserbase/browser-login-analysis.spec.ts @@ -0,0 +1,81 @@ +import { + analyzeDetectedLogin, + manualLoginAnalysis, + type LoginDetection, +} from './browser-login-analysis'; + +const base: LoginDetection = { + reachable: true, + hasPasswordField: false, + identifierType: 'unknown', + ssoProviders: [], + hasPasskey: false, + extraFields: [], +}; + +describe('analyzeDetectedLogin', () => { + it('recommends "ready" when a password field is present', () => { + const result = analyzeDetectedLogin({ + ...base, + hasPasswordField: true, + identifierType: 'email', + }); + expect(result.recommendation.category).toBe('ready'); + expect(result.detectedMethods).toContain('password'); + }); + + it('recommends check-ins for SSO-only sites', () => { + const result = analyzeDetectedLogin({ ...base, ssoProviders: ['Google'] }); + expect(result.recommendation.category).toBe('works_with_checkins'); + expect(result.detectedMethods).toContain('sso'); + }); + + it('recommends check-ins for passkey-only sites', () => { + const result = analyzeDetectedLogin({ ...base, hasPasskey: true }); + expect(result.recommendation.category).toBe('works_with_checkins'); + expect(result.detectedMethods).toContain('passkey'); + }); + + it('prefers the password path (ready) when both password and SSO exist', () => { + const result = analyzeDetectedLogin({ + ...base, + hasPasswordField: true, + ssoProviders: ['Google'], + }); + expect(result.recommendation.category).toBe('ready'); + expect(result.detectedMethods).toEqual( + expect.arrayContaining(['password', 'sso']), + ); + }); + + it('falls back to manual when nothing is detected', () => { + expect(analyzeDetectedLogin(base).recommendation.category).toBe('manual'); + }); + + it('falls back to manual when the page is unreachable, even with a password field', () => { + const result = analyzeDetectedLogin({ + ...base, + reachable: false, + hasPasswordField: true, + }); + expect(result.recommendation.category).toBe('manual'); + }); + + it('passes extra fields through unchanged', () => { + const result = analyzeDetectedLogin({ + ...base, + hasPasswordField: true, + extraFields: [{ label: 'Workspace URL' }], + }); + expect(result.extraFields).toEqual([{ label: 'Workspace URL' }]); + }); +}); + +describe('manualLoginAnalysis', () => { + it('is an unreachable, manual recommendation', () => { + const result = manualLoginAnalysis(); + expect(result.reachable).toBe(false); + expect(result.recommendation.category).toBe('manual'); + expect(result.detectedMethods).toEqual([]); + }); +}); diff --git a/apps/api/src/browserbase/browser-login-analysis.ts b/apps/api/src/browserbase/browser-login-analysis.ts new file mode 100644 index 0000000000..fb8039b569 --- /dev/null +++ b/apps/api/src/browserbase/browser-login-analysis.ts @@ -0,0 +1,115 @@ +import { z } from 'zod'; + +/** + * What we can reliably read from a vendor's *first* sign-in page. Note: the + * specific 2FA method (authenticator vs SMS vs push) usually isn't visible until + * after the identifier/password step, so it is resolved during live sign-in — not + * here. This detection is best-effort and always degrades to a manual fallback. + */ +export const loginDetectionSchema = z.object({ + reachable: z + .boolean() + .describe('Whether this looks like a real sign-in page we could read'), + hasPasswordField: z + .boolean() + .describe('Whether a password input is present on the page'), + identifierType: z + .enum(['email', 'username', 'either', 'unknown']) + .describe('What the first login field accepts'), + ssoProviders: z + .array(z.string()) + .describe( + 'Third-party sign-in buttons offered, e.g. Google, Microsoft, SSO', + ), + hasPasskey: z + .boolean() + .describe('Whether passkey / security-key sign-in is offered'), + extraFields: z + .array(z.object({ label: z.string() })) + .describe( + 'Other required fields before the password, e.g. company, workspace, subdomain', + ), +}); + +export type LoginDetection = z.infer; + +export type LoginMethod = 'password' | 'sso' | 'passkey'; + +export type LoginRecommendationCategory = + 'ready' | 'works_with_checkins' | 'manual'; + +export interface LoginAnalysis { + reachable: boolean; + detectedMethods: LoginMethod[]; + identifierType: LoginDetection['identifierType']; + extraFields: { label: string }[]; + recommendation: { + category: LoginRecommendationCategory; + headline: string; + detail: string; + }; +} + +const READY = { + category: 'ready' as const, + headline: "You're set.", + detail: + 'Sign in once — we capture what the scheduler needs to sign in on its own from then on.', +}; + +const CHECKINS = { + category: 'works_with_checkins' as const, + headline: 'This will work — with an occasional check-in.', + detail: + 'Runs reuse your session; when it expires we email you to sign in again. Runs pause, never fail silently.', +}; + +const MANUAL = { + category: 'manual' as const, + headline: "We couldn't detect a sign-in form here.", + detail: + "Make sure the URL is your login page (not the homepage) — or just sign in below and we'll take it from there.", +}; + +/** + * Turns raw page detection into a recommendation for the connect flow. Pure and + * deterministic so it can be unit-tested without a browser. + */ +export function analyzeDetectedLogin(detection: LoginDetection): LoginAnalysis { + const detectedMethods: LoginMethod[] = []; + if (detection.hasPasswordField) detectedMethods.push('password'); + if (detection.ssoProviders.length > 0) detectedMethods.push('sso'); + if (detection.hasPasskey) detectedMethods.push('passkey'); + + let recommendation: LoginAnalysis['recommendation'] = MANUAL; + if (detection.reachable && detection.hasPasswordField) { + // A password form means we can capture credentials + an authenticator key + // and run fully unattended (the 2FA specifics surface during sign-in). + recommendation = READY; + } else if ( + detection.reachable && + (detection.ssoProviders.length > 0 || detection.hasPasskey) + ) { + // SSO / passkey can't be replayed unattended — session reuse + human re-auth. + recommendation = CHECKINS; + } + + return { + reachable: detection.reachable, + detectedMethods, + identifierType: detection.identifierType, + extraFields: detection.extraFields, + recommendation, + }; +} + +export function manualLoginAnalysis(): LoginAnalysis { + return analyzeDetectedLogin({ + reachable: false, + hasPasswordField: false, + identifierType: 'unknown', + ssoProviders: [], + hasPasskey: false, + extraFields: [], + }); +} diff --git a/apps/api/src/browserbase/browser-login-analyzer.service.spec.ts b/apps/api/src/browserbase/browser-login-analyzer.service.spec.ts new file mode 100644 index 0000000000..b836ac80af --- /dev/null +++ b/apps/api/src/browserbase/browser-login-analyzer.service.spec.ts @@ -0,0 +1,90 @@ +import { BrowserLoginAnalyzerService } from './browser-login-analyzer.service'; +import type { BrowserbaseSessionService } from './browserbase-session.service'; +import type { LoginDetection } from './browser-login-analysis'; + +function makeSessions(extract: jest.Mock) { + const page = { goto: jest.fn().mockResolvedValue(undefined) }; + return { + createBrowserbaseContext: jest.fn().mockResolvedValue('ctx_1'), + createSessionWithContext: jest + .fn() + .mockResolvedValue({ sessionId: 'sess_1', liveViewUrl: '' }), + createStagehand: jest + .fn() + .mockResolvedValue({ + extract, + act: jest.fn().mockResolvedValue(undefined), + }), + ensureActivePage: jest.fn().mockResolvedValue(page), + safeCloseStagehand: jest.fn().mockResolvedValue(undefined), + closeSession: jest.fn().mockResolvedValue(undefined), + }; +} + +const passwordDetection: LoginDetection = { + reachable: true, + hasPasswordField: true, + identifierType: 'email', + ssoProviders: [], + hasPasskey: false, + extraFields: [], +}; + +describe('BrowserLoginAnalyzerService', () => { + beforeEach(() => jest.useFakeTimers()); + afterEach(() => jest.useRealTimers()); + + const run = async ( + sessions: ReturnType, + url: string, + ) => { + const service = new BrowserLoginAnalyzerService( + sessions as unknown as BrowserbaseSessionService, + ); + const promise = service.analyzeLogin(url); + await jest.runAllTimersAsync(); + return promise; + }; + + it('returns a ready recommendation and cleans up the session', async () => { + const extract = jest.fn().mockResolvedValue(passwordDetection); + const sessions = makeSessions(extract); + + const result = await run( + sessions, + 'https://app.datadoghq.com/account/login', + ); + + expect(result.recommendation.category).toBe('ready'); + expect(sessions.safeCloseStagehand).toHaveBeenCalledTimes(1); + expect(sessions.closeSession).toHaveBeenCalledWith('sess_1'); + }); + + it('falls back to manual when the page cannot be read, and still cleans up', async () => { + const extract = jest.fn().mockRejectedValue(new Error('page unreadable')); + const sessions = makeSessions(extract); + + const result = await run(sessions, 'https://weird.example.com'); + + expect(result.reachable).toBe(false); + expect(result.recommendation.category).toBe('manual'); + expect(sessions.closeSession).toHaveBeenCalledWith('sess_1'); + }); + + it('falls back to manual (no crash) when Browserbase is unavailable', async () => { + const sessions = makeSessions(jest.fn()); + sessions.createBrowserbaseContext.mockRejectedValue( + new Error('BROWSERBASE_API_KEY is missing'), + ); + + const result = await run( + sessions, + 'https://app.datadoghq.com/account/login', + ); + + expect(result.reachable).toBe(false); + expect(result.recommendation.category).toBe('manual'); + // No session was created, so nothing to close. + expect(sessions.closeSession).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/src/browserbase/browser-login-analyzer.service.ts b/apps/api/src/browserbase/browser-login-analyzer.service.ts new file mode 100644 index 0000000000..96b9f87f04 --- /dev/null +++ b/apps/api/src/browserbase/browser-login-analyzer.service.ts @@ -0,0 +1,81 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { BrowserbaseSessionService } from './browserbase-session.service'; +import { + analyzeDetectedLogin, + loginDetectionSchema, + manualLoginAnalysis, + type LoginAnalysis, +} from './browser-login-analysis'; +import { navigateToSignIn } from './browser-login-navigation'; + +const EXTRACT_PROMPT = + 'Look at this sign-in page. Report: is it a real login page we can read; is a ' + + 'password field present; does the first login field accept an email, a username, ' + + 'or either; which third-party sign-in buttons are offered (e.g. Google, Microsoft, ' + + 'Okta, SSO); is passkey / security-key sign-in offered; and list any other fields ' + + 'required before the password (e.g. company, workspace, subdomain).'; + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** + * Opens a vendor's sign-in page in a throwaway cloud browser and detects which + * login methods it supports, so the connect flow can recommend the most reliable + * path. Reads a public page only — no credentials involved. Always degrades to a + * manual fallback if the page can't be read. + */ +@Injectable() +export class BrowserLoginAnalyzerService { + private readonly logger = new Logger(BrowserLoginAnalyzerService.name); + + constructor( + private readonly sessions: BrowserbaseSessionService = new BrowserbaseSessionService(), + ) {} + + async analyzeLogin(url: string): Promise { + let sessionId: string | null = null; + let stagehand: Awaited< + ReturnType + > | null = null; + try { + // Session/context creation is inside the try so that an unavailable or + // unconfigured Browserbase (e.g. missing BROWSERBASE_API_KEY) degrades to + // the manual fallback instead of surfacing a 500 to the user. + const contextId = await this.sessions.createBrowserbaseContext(); + const session = await this.sessions.createSessionWithContext(contextId); + sessionId = session.sessionId; + + stagehand = await this.sessions.createStagehand(sessionId); + const page = await this.sessions.ensureActivePage(stagehand); + await page.goto(url, { waitUntil: 'domcontentloaded', timeoutMs: 30000 }); + await delay(1500); + + // Navigate to the actual sign-in page so the customer can paste any URL + // (a homepage, a dashboard) rather than the exact login page. + await navigateToSignIn(stagehand); + + const detection = await stagehand.extract( + EXTRACT_PROMPT, + loginDetectionSchema, + ); + return analyzeDetectedLogin(detection); + } catch (err) { + // A page we can't read (or a Browserbase hiccup) isn't an error the user + // needs to see — fall back to manual entry so the connect flow continues. + // Log the full error + stack so a misconfiguration is diagnosable and not + // silently hidden behind the manual fallback. + this.logger.warn('Login analysis failed; falling back to manual entry', { + url, + error: err instanceof Error ? err.message : String(err), + stack: err instanceof Error ? err.stack : undefined, + }); + return manualLoginAnalysis(); + } finally { + if (stagehand) await this.sessions.safeCloseStagehand(stagehand); + if (sessionId) { + await this.sessions + .closeSession(sessionId) + .catch(() => undefined /* best-effort cleanup */); + } + } + } +} diff --git a/apps/api/src/browserbase/browser-login-navigation.ts b/apps/api/src/browserbase/browser-login-navigation.ts new file mode 100644 index 0000000000..982c18adc5 --- /dev/null +++ b/apps/api/src/browserbase/browser-login-navigation.ts @@ -0,0 +1,50 @@ +type Stagehand = import('@browserbasehq/stagehand').Stagehand; + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** Current URL of the active page — a hint for settling; '' when unavailable. */ +function currentUrl(stagehand: Stagehand): string { + try { + const pages = stagehand.context?.pages?.() ?? []; + return pages[pages.length - 1]?.url() ?? ''; + } catch { + return ''; + } +} + +/** + * Wait until the URL stops changing, so a redirect chain (e.g. AWS bouncing from + * the homepage through signin.aws.amazon.com) finishes before the caller reads or + * fills the page. Bounded so a page that never settles can't hang the run. + */ +async function waitForUrlToSettle(stagehand: Stagehand): Promise { + let previous = currentUrl(stagehand); + for (let i = 0; i < 8; i += 1) { + await delay(700); + const now = currentUrl(stagehand); + if (now === previous) return; + previous = now; + } +} + +/** + * Best-effort navigation to the sign-in form. If the current page is a homepage + * or dashboard, the agent opens the "Sign in" link so the login form is on + * screen. Never throws — callers proceed on whatever page they end up on. Shared + * by login analysis and the automated sign-in so both start from the real form + * rather than, say, a marketing homepage. + */ +export async function navigateToSignIn(stagehand: Stagehand): Promise { + try { + await stagehand.act( + 'If this page is not already a login or sign-in page, find and open the ' + + '"Sign in", "Sign in to Console", or "Log in" link so the sign-in form is ' + + 'visible. If a sign-in form is already shown, do nothing.', + ); + // A sign-in link often kicks off a redirect chain; don't read/fill until it + // settles, or we'd act on the old (e.g. marketing homepage) page. + await waitForUrlToSettle(stagehand); + } catch { + // Ignore — detection / sign-in proceeds on the current page. + } +} diff --git a/apps/api/src/browserbase/browser-mfa-instructions.service.spec.ts b/apps/api/src/browserbase/browser-mfa-instructions.service.spec.ts new file mode 100644 index 0000000000..99bc4fcab9 --- /dev/null +++ b/apps/api/src/browserbase/browser-mfa-instructions.service.spec.ts @@ -0,0 +1,147 @@ +import { generateObject } from 'ai'; +import { BrowserMfaInstructionsService } from './browser-mfa-instructions.service'; + +jest.mock('ai', () => ({ generateObject: jest.fn() })); +jest.mock('@ai-sdk/anthropic', () => ({ anthropic: () => 'mock-model' })); + +const mockGenerate = generateObject as jest.MockedFunction; + +// Minimal shape the service reads from generateObject's result. +const asResult = (object: { steps: string[]; confident: boolean }) => + ({ object }) as unknown as Awaited>; + +const firecrawlOk = (markdown: string) => + ({ + ok: true, + json: async () => ({ + success: true, + data: { web: [{ url: 'https://docs.vendor.com/2fa', title: '2FA', markdown }] }, + }), + }) as unknown as Response; + +describe('BrowserMfaInstructionsService', () => { + let service: BrowserMfaInstructionsService; + let mockFetch: jest.Mock; + const originalKey = process.env.FIRECRAWL_API_KEY; + const originalFetch = global.fetch; + + beforeEach(() => { + jest.clearAllMocks(); + // Ungrounded by default (no web-search call) so the base cases are + // deterministic; grounding tests opt in by setting the key + mocking fetch. + delete process.env.FIRECRAWL_API_KEY; + mockFetch = jest.fn(); + global.fetch = mockFetch as unknown as typeof fetch; + service = new BrowserMfaInstructionsService(); + }); + + afterEach(() => { + process.env.FIRECRAWL_API_KEY = originalKey; + global.fetch = originalFetch; + }); + + it('returns generated steps when the model is confident', async () => { + mockGenerate.mockResolvedValueOnce( + asResult({ steps: ['Open Security', 'Add authenticator'], confident: true }), + ); + + const result = await service.getInstructions('https://github.com/login'); + + expect(result.hostname).toBe('github.com'); + expect(result.source).toBe('generated'); + expect(result.confident).toBe(true); + expect(result.grounded).toBe(false); + expect(result.steps).toEqual(['Open Security', 'Add authenticator']); + expect(result.tips.length).toBeGreaterThan(0); + // No key → no web-search call. + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it('strips markdown emphasis the model emits', async () => { + mockGenerate.mockResolvedValueOnce( + asResult({ + steps: ['Open **Settings**', 'Click `Password and authentication`'], + confident: true, + }), + ); + + const result = await service.getInstructions('github.com'); + + expect(result.steps).toEqual([ + 'Open Settings', + 'Click Password and authentication', + ]); + }); + + it('falls back to universal steps when the model is not confident', async () => { + mockGenerate.mockResolvedValueOnce( + asResult({ steps: ['A guessed step'], confident: false }), + ); + + const result = await service.getInstructions('obscure-vendor.example.com'); + + expect(result.source).toBe('fallback'); + expect(result.confident).toBe(false); + expect(result.grounded).toBe(false); + expect(result.steps).not.toContain('A guessed step'); + expect(result.steps.length).toBeGreaterThan(0); + }); + + it('falls back when generation throws', async () => { + mockGenerate.mockRejectedValueOnce(new Error('model unavailable')); + + const result = await service.getInstructions('aws.amazon.com'); + + expect(result.source).toBe('fallback'); + expect(result.steps.length).toBeGreaterThan(0); + }); + + it('caches per normalized host — a full URL and bare host share one entry', async () => { + mockGenerate.mockResolvedValueOnce( + asResult({ steps: ['Open Security'], confident: true }), + ); + + const first = await service.getInstructions('https://console.aws.amazon.com/iam'); + const second = await service.getInstructions('console.aws.amazon.com'); + + expect(first.hostname).toBe('console.aws.amazon.com'); + expect(second).toEqual(first); + // Second call served from cache → the model was only invoked once. + expect(mockGenerate).toHaveBeenCalledTimes(1); + }); + + it('grounds the prompt in vendor docs when Firecrawl returns results', async () => { + process.env.FIRECRAWL_API_KEY = 'test-key'; + mockFetch.mockResolvedValueOnce( + firecrawlOk('Go to Settings > Security > Add authenticator app and click "setup key".'), + ); + mockGenerate.mockResolvedValueOnce( + asResult({ steps: ['Open Settings', 'Add authenticator app'], confident: true }), + ); + + const result = await service.getInstructions('github.com'); + + expect(mockFetch).toHaveBeenCalledTimes(1); + expect(result.grounded).toBe(true); + expect(result.source).toBe('generated'); + // The docs were injected into the generation prompt. + const promptArg = mockGenerate.mock.calls[0][0].prompt as string; + expect(promptArg).toContain('CURRENT VENDOR DOCS'); + expect(promptArg).toContain('setup key'); + }); + + it('generates ungrounded when Firecrawl fails', async () => { + process.env.FIRECRAWL_API_KEY = 'test-key'; + mockFetch.mockRejectedValueOnce(new Error('firecrawl down')); + mockGenerate.mockResolvedValueOnce( + asResult({ steps: ['Open Security'], confident: true }), + ); + + const result = await service.getInstructions('github.com'); + + expect(result.grounded).toBe(false); + expect(result.source).toBe('generated'); + const promptArg = mockGenerate.mock.calls[0][0].prompt as string; + expect(promptArg).not.toContain('CURRENT VENDOR DOCS'); + }); +}); diff --git a/apps/api/src/browserbase/browser-mfa-instructions.service.ts b/apps/api/src/browserbase/browser-mfa-instructions.service.ts new file mode 100644 index 0000000000..6defbca960 --- /dev/null +++ b/apps/api/src/browserbase/browser-mfa-instructions.service.ts @@ -0,0 +1,271 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { anthropic } from '@ai-sdk/anthropic'; +import { generateObject } from 'ai'; +import { z } from 'zod'; +import { normalizeHostnameFromUrl } from './browserbase-url'; + +// Guidance is plain natural language with no SDK-call shape to validate, so the +// cheaper/faster model is the right fit — same choice as the manual-steps +// fallback in ai-remediation.service. +const MODEL = anthropic('claude-sonnet-4-6'); + +// A vendor's MFA setup UI rarely changes, so a day keeps guidance fresh while +// making all-but-the-first request for a vendor instant and free. In-memory is +// deliberate: instructions are public and cheap to regenerate, so a per-instance +// TTL cache avoids a DB migration; promoting this to a shared table later is a +// drop-in swap behind `getInstructions`. +const CACHE_TTL_MS = 24 * 60 * 60 * 1000; + +// Grounding: before generating, pull the vendor's CURRENT help docs via web +// search and feed them to the model, so steps track the live UI rather than the +// model's training snapshot. Best-effort — a missing key/failure/empty result +// just falls back to ungrounded generation (still confidence-gated). +const FIRECRAWL_SEARCH_URL = 'https://api.firecrawl.dev/v2/search'; +const GROUNDING_RESULT_LIMIT = 3; +const GROUNDING_TIMEOUT_MS = 20_000; +const PER_DOC_CHARS = 2_500; +const TOTAL_DOC_CHARS = 6_000; + +interface FirecrawlSearchResult { + url?: string; + title?: string; + markdown?: string | null; +} + +interface FirecrawlSearchResponse { + success?: boolean; + data?: { web?: FirecrawlSearchResult[] }; +} + +const instructionSchema = z.object({ + steps: z + .array(z.string().min(1)) + .describe( + 'Ordered, concrete steps for a user of THIS vendor to add a NEW authenticator (TOTP) app and reveal its manual "setup key" / "secret key". Each step is one short action. 3-7 steps.', + ), + confident: z + .boolean() + .describe( + "true ONLY if these steps reflect the vendor's CURRENT, real settings UI. false if you are guessing menu/button names or do not recognize the vendor — the caller then shows a generic fallback instead of shaky specifics.", + ), +}); + +export interface MfaInstructions { + hostname: string; + steps: string[]; + tips: string[]; + confident: boolean; + /** Whether the steps were grounded in the vendor's current help docs. */ + grounded: boolean; + /** ISO timestamp the steps were produced — powers the "checked on" trust line. */ + checkedAt: string; + source: 'generated' | 'fallback'; +} + +interface CacheEntry { + value: MfaInstructions; + expiresAt: number; +} + +// Always-true, vendor-agnostic pointers, safe to show alongside any steps. These +// are universal TOTP facts, NOT per-vendor hardcode. +const UNIVERSAL_TIPS = [ + 'When the vendor shows a QR code, pick "Can\'t scan?" / "Enter this code manually" and copy that long key (the setup key) — not the rotating 6-digit code.', + 'Where the vendor allows more than one device, add a dedicated authenticator just for this automation so it can be revoked without touching your personal one.', +]; + +// The single generic safety net shown when generation is not confident. One +// universal instruction — not per-vendor hardcode — so we never show invented, +// wrong steps. +const UNIVERSAL_STEPS = [ + 'Sign in to the vendor and open your account Security / Two-factor authentication settings.', + 'Choose to add an authenticator app (TOTP).', + 'When the QR code appears, select "Can\'t scan?" / "Enter this code manually" to reveal the setup key.', + 'Copy that setup key and paste it into Comp AI.', +]; + +const SYSTEM_PROMPT = `You help a user turn on an authenticator app (TOTP) for a third-party SaaS vendor so an automation can generate their 2FA codes. + +Given only the vendor's hostname, produce the exact steps to: +1. Open that vendor's account security / two-factor settings, +2. Add a NEW authenticator app (TOTP — NOT SMS, NOT email codes, NOT a security key/passkey), +3. Reveal the manual setup key (vendors usually show a QR plus a "can't scan / enter code manually" option that reveals a long alphanumeric key). + +RULES: +- If CURRENT VENDOR DOCS are provided, treat them as the source of truth for the vendor's live UI and base the steps on them. +- Otherwise, base the steps on the vendor's CURRENT, real UI. Use real menu/button names only when you are confident of them. +- Do NOT invent specific button or menu names you are unsure about. +- Only cover authenticator-app (TOTP) setup — never SMS, email, or hardware key/passkey. +- One short action per step. 3-7 steps. +- Write each step in PLAIN TEXT. Do NOT use markdown, asterisks, backticks, or bold — the UI renders raw text. +- Set confident=true ONLY if the steps reflect this vendor's actual current UI (from the provided docs, or your solid knowledge of it). If you do not recognize the vendor or are unsure of the path, set confident=false.`; + +/** Strip markdown emphasis the model sometimes emits (`**bold**`, `__`, backticks). */ +function stripEmphasis(text: string): string { + return text.replace(/\*\*/g, '').replace(/__/g, '').replace(/`/g, '').trim(); +} + +function buildPrompt(hostname: string, grounding: string | null): string { + const base = `Vendor hostname: ${hostname} + +Write the steps for a user of this exact vendor to add an authenticator app and reveal its manual setup key.`; + + if (grounding) { + return `${base} + +CURRENT VENDOR DOCS (from a live web search — treat as the source of truth for the vendor's current UI): +""" +${grounding} +""" + +Base your steps on these docs. If they clearly describe adding an authenticator app / TOTP and revealing the manual setup key, set confident=true.`; + } + + return `${base} If you do not recognize this vendor or are not confident of its current settings UI, set confident=false.`; +} + +/** + * Produces per-vendor, human-readable instructions for obtaining an authenticator + * (TOTP) setup key, so users can hand Comp AI the seed for unattended 2FA. Steps + * are AI-generated (no per-vendor hardcode), confidence-gated to a universal + * fallback, and cached per hostname. + */ +@Injectable() +export class BrowserMfaInstructionsService { + private readonly logger = new Logger(BrowserMfaInstructionsService.name); + private readonly cache = new Map(); + + async getInstructions(rawHost: string): Promise { + const hostname = this.normalizeHost(rawHost); + + const cached = this.cache.get(hostname); + if (cached && cached.expiresAt > Date.now()) return cached.value; + + let value: MfaInstructions; + try { + value = await this.generate(hostname); + } catch (err) { + this.logger.warn( + `MFA instruction generation failed for ${hostname}; using fallback. ${ + err instanceof Error ? err.message : String(err) + }`, + ); + value = this.fallback(hostname); + } + + this.cache.set(hostname, { value, expiresAt: Date.now() + CACHE_TTL_MS }); + return value; + } + + private async generate(hostname: string): Promise { + const grounding = await this.fetchGroundingDocs(hostname); + const grounded = Boolean(grounding); + + const { object } = await generateObject({ + model: MODEL, + schema: instructionSchema, + system: SYSTEM_PROMPT, + prompt: buildPrompt(hostname, grounding), + temperature: 0.2, + }); + + // Don't show shaky, possibly-invented steps — fall back to the universal + // instruction whenever the model isn't confident (or returned nothing). + if (!object.confident || object.steps.length === 0) { + this.logger.log( + `MFA instructions for ${hostname}: not confident → fallback (grounded=${grounded})`, + ); + return this.fallback(hostname); + } + + this.logger.log( + `MFA instructions for ${hostname}: generated ${object.steps.length} step(s) (grounded=${grounded})`, + ); + return { + hostname, + steps: object.steps.map(stripEmphasis).filter((step) => step.length > 0), + tips: UNIVERSAL_TIPS, + confident: true, + grounded, + checkedAt: new Date().toISOString(), + source: 'generated', + }; + } + + private fallback(hostname: string): MfaInstructions { + return { + hostname, + steps: UNIVERSAL_STEPS, + tips: UNIVERSAL_TIPS, + confident: false, + grounded: false, + checkedAt: new Date().toISOString(), + source: 'fallback', + }; + } + + /** + * Best-effort: pull the vendor's current help docs via Firecrawl web search to + * ground the generated steps in the live UI. Returns null (never throws) when + * the key is missing, the request fails/times out, or nothing useful is found — + * generation then proceeds ungrounded. + */ + private async fetchGroundingDocs(hostname: string): Promise { + const apiKey = process.env.FIRECRAWL_API_KEY; + if (!apiKey) return null; + + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), GROUNDING_TIMEOUT_MS); + try { + const response = await fetch(FIRECRAWL_SEARCH_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + query: `${hostname} set up an authenticator app (TOTP) for two-factor authentication and reveal the manual setup key / secret key`, + limit: GROUNDING_RESULT_LIMIT, + sources: [{ type: 'web' }], + scrapeOptions: { formats: ['markdown'], onlyMainContent: true }, + }), + signal: controller.signal, + }); + if (!response.ok) return null; + + const body = (await response.json()) as FirecrawlSearchResponse; + const docs = (body?.data?.web ?? []) + .map((result) => { + const markdown = (result.markdown ?? '').trim(); + if (!markdown) return null; + return `# ${result.title ?? result.url ?? 'Result'}\n${ + result.url ?? '' + }\n${markdown.slice(0, PER_DOC_CHARS)}`; + }) + .filter((doc): doc is string => Boolean(doc)); + + if (docs.length === 0) return null; + return docs.join('\n\n---\n\n').slice(0, TOTAL_DOC_CHARS); + } catch (err) { + this.logger.warn( + `Firecrawl grounding failed for ${hostname}: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + return null; + } finally { + clearTimeout(timeout); + } + } + + /** Accepts a full URL or a bare hostname; always returns a normalized host. */ + private normalizeHost(rawHost: string): string { + const trimmed = rawHost.trim(); + const withScheme = /^https?:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}`; + try { + return normalizeHostnameFromUrl(withScheme); + } catch { + return trimmed.toLowerCase(); + } + } +} diff --git a/apps/api/src/browserbase/browserbase-session.service.ts b/apps/api/src/browserbase/browserbase-session.service.ts index 76e62ca6ee..bd61557917 100644 --- a/apps/api/src/browserbase/browserbase-session.service.ts +++ b/apps/api/src/browserbase/browserbase-session.service.ts @@ -10,15 +10,71 @@ import { isNoPageError } from './run-error-formatter'; type Stagehand = import('@browserbasehq/stagehand').Stagehand; -const BROWSER_WIDTH = 1440; -const BROWSER_HEIGHT = 900; -const STAGEHAND_MODEL = 'anthropic/claude-sonnet-4-6'; +export interface BrowserViewport { + width: number; + height: number; +} + +// A larger native viewport keeps unattended evidence screenshots sharp: our +// panels are well under 1920px wide, so the viewer downscales rather than +// upscaling — the latter is what looked soft, especially on HiDPI displays. +// Browserbase exposes no device-pixel-ratio setting, so viewport size is the +// only lever for capture sharpness. This is the default. +export const CAPTURE_VIEWPORT: BrowserViewport = { width: 1920, height: 1080 }; + +// A smaller viewport for human-facing sessions (sign-in, take-over, SSO, +// reconnect): the page renders larger in the embedded live view, so forms are +// easier to read and fill. Capture runs stay on CAPTURE_VIEWPORT. +export const INTERACTIVE_VIEWPORT: BrowserViewport = { width: 1280, height: 800 }; +// Model behind extract()/act() (reading pages, verdicts, form fills). Separate +// from the navigation (CUA) model and configurable via env; default unchanged. +const STAGEHAND_MODEL = + process.env.BROWSERBASE_STAGEHAND_MODEL || 'anthropic/claude-sonnet-4-6'; const BROWSERBASE_API_MAX_ATTEMPTS = 3; const BROWSERBASE_RETRY_DELAYS_MS = [250, 750]; const BROWSERBASE_DEFAULT_HEADERS = { 'accept-encoding': 'identity' }; const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +/** Hostname of a URL, or null if it can't be parsed (used to match live tabs). */ +function hostnameOrNull(url: string): string | null { + try { + return new URL(url).hostname; + } catch { + return null; + } +} + +/** A live-view page as returned by Browserbase `sessions.debug()`. */ +interface LiveViewPage { + url: string; + debuggerFullscreenUrl: string; +} + +/** + * Pick the live-view URL to show: the tab matching `matchUrl` (the page the AI + * is on) by exact URL, else by hostname, else the newest tab, else the + * session-level view. Pure so the tab-selection can be unit-tested. + */ +export function selectLiveViewUrl( + debug: { debuggerFullscreenUrl?: string; pages?: LiveViewPage[] }, + matchUrl?: string, +): string | null { + const pages = debug.pages ?? []; + if (pages.length === 0) return debug.debuggerFullscreenUrl ?? null; + + if (matchUrl) { + const exact = pages.find((page) => page.url === matchUrl); + if (exact) return exact.debuggerFullscreenUrl; + const host = hostnameOrNull(matchUrl); + const byHost = + host !== null ? pages.find((page) => hostnameOrNull(page.url) === host) : undefined; + if (byHost) return byHost.debuggerFullscreenUrl; + } + // Newest tab is usually the just-opened sign-in. + return pages[pages.length - 1]?.debuggerFullscreenUrl ?? debug.debuggerFullscreenUrl ?? null; +} + @Injectable() export class BrowserbaseSessionService { private readonly logger = new Logger(BrowserbaseSessionService.name); @@ -48,6 +104,7 @@ export class BrowserbaseSessionService { async createSessionWithContext( contextId: string, + viewport: BrowserViewport = CAPTURE_VIEWPORT, ): Promise<{ sessionId: string; liveViewUrl: string }> { const bb = this.getBrowserbase(); @@ -63,13 +120,13 @@ export class BrowserbaseSessionService { }, fingerprint: { screen: { - maxHeight: BROWSER_HEIGHT, - maxWidth: BROWSER_WIDTH, - minHeight: BROWSER_HEIGHT, - minWidth: BROWSER_WIDTH, + maxHeight: viewport.height, + maxWidth: viewport.width, + minHeight: viewport.height, + minWidth: viewport.width, }, }, - viewport: { width: BROWSER_WIDTH, height: BROWSER_HEIGHT }, + viewport: { width: viewport.width, height: viewport.height }, }, keepAlive: true, }), @@ -95,6 +152,23 @@ export class BrowserbaseSessionService { } } + /** + * Live-view URL for a specific tab, so the UI can follow the AI when a vendor + * opens its sign-in in a new tab (e.g. AWS from its homepage). Prefers the tab + * matching `matchUrl` (the page the AI is on), falls back to the newest tab, + * then the session-level view. Returns null if the session has no pages. + */ + async getActivePageLiveViewUrl( + sessionId: string, + matchUrl?: string, + ): Promise { + const debug = await this.withBrowserbaseRetry({ + operationName: 'session live view lookup', + operation: () => this.getBrowserbase().sessions.debug(sessionId), + }); + return selectLiveViewUrl(debug, matchUrl); + } + async closeSession(sessionId: string): Promise { await this.withBrowserbaseRetry({ operationName: 'session close', @@ -215,6 +289,7 @@ export class BrowserbaseSessionService { try { await stagehand.init(); + await this.calmPageMotion(stagehand); return stagehand; } catch (error) { await this.safeCloseStagehand(stagehand); @@ -224,6 +299,33 @@ export class BrowserbaseSessionService { }); } + /** + * Inject "reduce motion" into every page of the session so a vendor's own + * looping animations (heavy hero backgrounds, marquees, spinners) don't churn + * the live-view stream — that constant repainting is what looks laggy. Also + * steadies evidence screenshots. Passed as a string so the browser-side + * `document` isn't type-checked in this Node file. Best-effort; a calmer live + * view is a nice-to-have, not critical to the run. + */ + private async calmPageMotion(stagehand: Stagehand): Promise { + const script = `(() => { + const css = '*,*::before,*::after{animation-duration:0.001ms!important;animation-delay:0ms!important;animation-iteration-count:1!important;transition-duration:0.001ms!important;transition-delay:0ms!important;scroll-behavior:auto!important}'; + const style = document.createElement('style'); + style.setAttribute('data-comp-reduce-motion', ''); + style.textContent = css; + const attach = () => (document.head || document.documentElement).appendChild(style); + if (document.head) attach(); + else document.addEventListener('DOMContentLoaded', attach, { once: true }); + })()`; + try { + await stagehand.context.addInitScript(script); + } catch (err) { + this.logger.warn('Could not inject reduced-motion styles (ignored)', { + error: err instanceof Error ? err.message : String(err), + }); + } + } + async safeCloseStagehand(stagehand: Stagehand): Promise { try { await stagehand.close(); diff --git a/apps/api/src/browserbase/browserbase.controller.ts b/apps/api/src/browserbase/browserbase.controller.ts index a02d1fb69c..f361c3d500 100644 --- a/apps/api/src/browserbase/browserbase.controller.ts +++ b/apps/api/src/browserbase/browserbase.controller.ts @@ -25,20 +25,28 @@ import { PermissionGuard } from '../auth/permission.guard'; import { RequirePermission } from '../auth/require-permission.decorator'; import { BrowserbaseService } from './browserbase.service'; import { + AnalyzeLoginDto, + AnalyzeLoginResponseDto, AuthStatusResponseDto, BrowserAutomationResponseDto, BrowserAutomationRunResponseDto, CheckAuthDto, CloseSessionDto, ContextResponseDto, + CreateBrowserAutomationDraftDto, CreateBrowserAutomationDto, CreateSessionDto, ExecuteAutomationSessionDto, NavigateToUrlDto, RunAutomationResponseDto, SessionResponseDto, + SetTaskScheduleDto, + TestInstructionDto, + TestInstructionResponseDto, + UpdateBrowserAutomationDraftDto, UpdateBrowserAutomationDto, } from './dto/browserbase.dto'; +import { TaskFrequency } from '@db'; @ApiTags('Browserbase') @Controller({ path: 'browserbase', version: '1' }) @@ -47,6 +55,23 @@ import { export class BrowserbaseController { constructor(private readonly browserbaseService: BrowserbaseService) {} + // ===== Login Analysis ===== + + @Post('analyze-login') + @RequirePermission('integration', 'create') + @ApiOperation({ + summary: 'Analyze a vendor sign-in page', + description: + 'Opens the vendor sign-in page in a throwaway cloud browser and detects which login methods it supports, so the connect flow can recommend the most reliable setup. Reads a public page only — no credentials. Always degrades to a manual fallback.', + }) + @ApiBody({ type: AnalyzeLoginDto }) + @ApiResponse({ status: 201, type: AnalyzeLoginResponseDto }) + async analyzeLogin( + @Body() dto: AnalyzeLoginDto, + ): Promise { + return this.browserbaseService.analyzeLogin(dto.url); + } + // ===== Organization Context ===== @Post('org-context') @@ -101,9 +126,11 @@ export class BrowserbaseController { type: SessionResponseDto, }) async createSession( + @OrganizationId() organizationId: string, @Body() dto: CreateSessionDto, ): Promise { - return await this.browserbaseService.createSessionWithContext( + return await this.browserbaseService.createSessionForOrg( + organizationId, dto.contextId, ); } @@ -118,9 +145,13 @@ export class BrowserbaseController { description: 'Session closed', }) async closeSession( + @OrganizationId() organizationId: string, @Body() dto: CloseSessionDto, ): Promise<{ success: boolean }> { - await this.browserbaseService.closeSession(dto.sessionId); + await this.browserbaseService.closeSessionForOrg( + organizationId, + dto.sessionId, + ); return { success: true }; } @@ -137,9 +168,14 @@ export class BrowserbaseController { description: 'Navigation result', }) async navigateToUrl( + @OrganizationId() organizationId: string, @Body() dto: NavigateToUrlDto, ): Promise<{ success: boolean; error?: string }> { - return await this.browserbaseService.navigateToUrl(dto.sessionId, dto.url); + return await this.browserbaseService.navigateToUrlForOrg( + organizationId, + dto.sessionId, + dto.url, + ); } @Post('check-auth') @@ -153,8 +189,12 @@ export class BrowserbaseController { description: 'Auth status', type: AuthStatusResponseDto, }) - async checkAuth(@Body() dto: CheckAuthDto): Promise { - return await this.browserbaseService.checkLoginStatus( + async checkAuth( + @OrganizationId() organizationId: string, + @Body() dto: CheckAuthDto, + ): Promise { + return await this.browserbaseService.checkLoginStatusForOrg( + organizationId, dto.sessionId, dto.url, ); @@ -182,6 +222,29 @@ export class BrowserbaseController { )) as BrowserAutomationResponseDto; } + @Post('automations/test') + @RequirePermission('task', 'update') + @ApiOperation({ + summary: 'Test an instruction before saving', + description: + 'Runs a not-yet-saved instruction against the connection’s live session so the user can watch it work before committing it to the schedule. Nothing is persisted. Returns a run handle to subscribe to for live steps and the final result.', + }) + @ApiBody({ type: TestInstructionDto }) + @ApiResponse({ status: 201, type: TestInstructionResponseDto }) + async testInstruction( + @OrganizationId() organizationId: string, + @Body() dto: TestInstructionDto, + ): Promise { + return this.browserbaseService.testInstruction({ + organizationId, + taskId: dto.taskId, + profileId: dto.profileId, + targetUrl: dto.targetUrl, + instruction: dto.instruction, + evaluationCriteria: dto.evaluationCriteria, + }); + } + @Get('automations/task/:taskId') @RequirePermission('task', 'read') @ApiOperation({ @@ -247,6 +310,26 @@ export class BrowserbaseController { )) as BrowserAutomationResponseDto; } + @Patch('automations/task/:taskId/schedule') + @RequirePermission('task', 'update') + @ApiOperation({ + summary: 'Set the schedule for every browser automation on a task', + }) + @ApiParam({ name: 'taskId', description: 'Task ID' }) + @ApiResponse({ status: 200, description: 'Task schedule updated' }) + async setTaskSchedule( + @Param('taskId') taskId: string, + @OrganizationId() organizationId: string, + @Body() dto: SetTaskScheduleDto, + ): Promise<{ success: boolean; scheduleFrequency: TaskFrequency }> { + const result = await this.browserbaseService.setTaskSchedule( + taskId, + dto.scheduleFrequency, + organizationId, + ); + return { success: true, scheduleFrequency: result.scheduleFrequency }; + } + @Delete('automations/:automationId') @RequirePermission('task', 'delete') @ApiOperation({ @@ -268,6 +351,55 @@ export class BrowserbaseController { return { success: true }; } + // ===== Automation Drafts (in-progress, unsaved) ===== + + @Get('automations/task/:taskId/drafts') + @RequirePermission('task', 'read') + @ApiOperation({ summary: 'List in-progress automation drafts for a task' }) + @ApiParam({ name: 'taskId', description: 'Task ID' }) + async listDrafts( + @OrganizationId() organizationId: string, + @Param('taskId') taskId: string, + ) { + return this.browserbaseService.listAutomationDrafts(taskId, organizationId); + } + + @Post('automation-drafts') + @RequirePermission('task', 'create') + @ApiOperation({ summary: 'Create an in-progress automation draft' }) + @ApiBody({ type: CreateBrowserAutomationDraftDto }) + async createDraft( + @OrganizationId() organizationId: string, + @Body() dto: CreateBrowserAutomationDraftDto, + ) { + return this.browserbaseService.createAutomationDraft(dto, organizationId); + } + + @Patch('automation-drafts/:draftId') + @RequirePermission('task', 'update') + @ApiOperation({ summary: 'Autosave an in-progress automation draft' }) + @ApiParam({ name: 'draftId', description: 'Draft ID' }) + @ApiBody({ type: UpdateBrowserAutomationDraftDto }) + async updateDraft( + @OrganizationId() organizationId: string, + @Param('draftId') draftId: string, + @Body() dto: UpdateBrowserAutomationDraftDto, + ) { + return this.browserbaseService.updateAutomationDraft(draftId, dto, organizationId); + } + + @Delete('automation-drafts/:draftId') + @RequirePermission('task', 'delete') + @ApiOperation({ summary: 'Discard an in-progress automation draft' }) + @ApiParam({ name: 'draftId', description: 'Draft ID' }) + async deleteDraft( + @OrganizationId() organizationId: string, + @Param('draftId') draftId: string, + ): Promise<{ success: boolean }> { + await this.browserbaseService.deleteAutomationDraft(draftId, organizationId); + return { success: true }; + } + // ===== Automation Execution ===== @Post('automations/:automationId/start-live') @@ -320,6 +452,12 @@ export class BrowserbaseController { error?: string; needsReauth?: boolean; }> { + // The session is client-supplied — confirm it belongs to this org before we + // drive an automation on it (cross-tenant IDOR guard). + await this.browserbaseService.assertSessionOwnedByOrg( + organizationId, + body.sessionId, + ); return await this.browserbaseService.executeAutomationOnSession( automationId, body.runId, @@ -328,6 +466,35 @@ export class BrowserbaseController { ); } + @Post('automations/:automationId/execute-live') + @RequirePermission('task', 'update') + @ApiOperation({ + summary: 'Execute automation on a session with live step streaming', + description: + 'Runs the automation on a pre-created session as a background task so the ' + + 'live view can stream the AI’s steps. Returns a run handle to subscribe to.', + }) + @ApiParam({ name: 'automationId', description: 'Automation ID' }) + @ApiBody({ type: ExecuteAutomationSessionDto }) + @ApiResponse({ status: 200, description: 'Run handle for realtime steps' }) + async executeAutomationLive( + @Param('automationId') automationId: string, + @Body() body: ExecuteAutomationSessionDto, + @OrganizationId() organizationId: string, + ): Promise<{ runId: string; publicAccessToken: string }> { + // The session is client-supplied — confirm it belongs to this org first. + await this.browserbaseService.assertSessionOwnedByOrg( + organizationId, + body.sessionId, + ); + return await this.browserbaseService.startLiveAutomationExecution({ + automationId, + runId: body.runId, + sessionId: body.sessionId, + organizationId, + }); + } + @Post('automations/:automationId/run') @RequirePermission('task', 'update') @ApiOperation({ diff --git a/apps/api/src/browserbase/browserbase.module.ts b/apps/api/src/browserbase/browserbase.module.ts index 2346ad2130..c7835b2014 100644 --- a/apps/api/src/browserbase/browserbase.module.ts +++ b/apps/api/src/browserbase/browserbase.module.ts @@ -6,12 +6,16 @@ import { BrowserAuthProfilesController } from './browser-auth-profiles.controlle import { BrowserAuthProfileContextService } from './browser-auth-profile-context.service'; import { BrowserAuthProfileService } from './browser-auth-profile.service'; import { BrowserEvidenceRunnerService } from './browser-evidence-runner.service'; +import { BrowserInstructionTestService } from './browser-instruction-test.service'; import { BrowserbaseController } from './browserbase.controller'; import { BrowserbaseOrgContextService } from './browserbase-org-context.service'; import { BrowserbaseScreenshotService } from './browserbase-screenshot.service'; import { BrowserbaseSessionService } from './browserbase-session.service'; import { BrowserbaseService } from './browserbase.service'; +import { BrowserCredentialSigninService } from './browser-credential-signin.service'; import { BrowserCredentialStorageService } from './browser-credential-storage.service'; +import { BrowserLoginAnalyzerService } from './browser-login-analyzer.service'; +import { BrowserMfaInstructionsService } from './browser-mfa-instructions.service'; import { BROWSER_CREDENTIAL_VAULT_ADAPTER } from './credential-vault'; import { resolveBrowserCredentialVaultAdapter } from './browser-credential-vault.factory'; import { AuthModule } from '../auth/auth.module'; @@ -30,7 +34,11 @@ import { AuthModule } from '../auth/auth.module'; BrowserbaseOrgContextService, BrowserbaseScreenshotService, BrowserEvidenceRunnerService, + BrowserInstructionTestService, + BrowserCredentialSigninService, BrowserCredentialStorageService, + BrowserLoginAnalyzerService, + BrowserMfaInstructionsService, { provide: BROWSER_CREDENTIAL_VAULT_ADAPTER, useFactory: resolveBrowserCredentialVaultAdapter, diff --git a/apps/api/src/browserbase/browserbase.service.spec.ts b/apps/api/src/browserbase/browserbase.service.spec.ts index 62e7cc16b2..53d6917cdb 100644 --- a/apps/api/src/browserbase/browserbase.service.spec.ts +++ b/apps/api/src/browserbase/browserbase.service.spec.ts @@ -7,6 +7,7 @@ import { BrowserAutomationRunStoreService } from './browser-automation-run-store import { BrowserAuthProfileContextService } from './browser-auth-profile-context.service'; import { BrowserAuthProfileService } from './browser-auth-profile.service'; import { BrowserCredentialStorageService } from './browser-credential-storage.service'; +import { BrowserLoginAnalyzerService } from './browser-login-analyzer.service'; import { BrowserEvidenceRunnerService } from './browser-evidence-runner.service'; import { BrowserbaseOrgContextService } from './browserbase-org-context.service'; import { BrowserbaseScreenshotService } from './browserbase-screenshot.service'; @@ -15,24 +16,36 @@ import { BrowserbaseService } from './browserbase.service'; import { BROWSER_CREDENTIAL_VAULT_ADAPTER } from './credential-vault'; import { resolveBrowserCredentialVaultAdapter } from './browser-credential-vault.factory'; -jest.mock('@db', () => ({ - db: { +jest.mock('@db', () => { + const db = { browserAutomationRun: { findUnique: jest.fn(), }, browserAutomation: { create: jest.fn(), update: jest.fn(), + updateMany: jest.fn(), + findFirst: jest.fn(), }, - }, - TaskFrequency: { - daily: 'daily', - weekly: 'weekly', - monthly: 'monthly', - quarterly: 'quarterly', - yearly: 'yearly', - }, -})); + browserAutomationStep: { + deleteMany: jest.fn(), + updateMany: jest.fn(), + create: jest.fn(), + }, + // Steps updates run in a transaction; pass the same mock through as `tx`. + $transaction: jest.fn((fn: (tx: unknown) => unknown) => fn(db)), + }; + return { + db, + TaskFrequency: { + daily: 'daily', + weekly: 'weekly', + monthly: 'monthly', + quarterly: 'quarterly', + yearly: 'yearly', + }, + }; +}); jest.mock('@/app/s3', () => ({ getSignedUrl: jest.fn().mockResolvedValue('https://s3.example.com/signed'), @@ -61,6 +74,7 @@ describe('BrowserbaseService.getScreenshotRedirectUrl', () => { BrowserbaseScreenshotService, BrowserEvidenceRunnerService, BrowserCredentialStorageService, + BrowserLoginAnalyzerService, { provide: BROWSER_CREDENTIAL_VAULT_ADAPTER, useFactory: resolveBrowserCredentialVaultAdapter, @@ -184,6 +198,7 @@ describe('BrowserbaseService schedule frequency passthrough', () => { BrowserbaseScreenshotService, BrowserEvidenceRunnerService, BrowserCredentialStorageService, + BrowserLoginAnalyzerService, { provide: BROWSER_CREDENTIAL_VAULT_ADAPTER, useFactory: resolveBrowserCredentialVaultAdapter, @@ -256,4 +271,105 @@ describe('BrowserbaseService schedule frequency passthrough', () => { const call = (db.browserAutomation.update as jest.Mock).mock.calls[0][0]; expect(call.data).not.toHaveProperty('scheduleFrequency'); }); + + it('inherits the task cadence for a new automation when none is given', async () => { + (db.browserAutomation.findFirst as jest.Mock).mockResolvedValue({ + scheduleFrequency: TaskFrequency.weekly, + }); + (db.browserAutomation.create as jest.Mock).mockResolvedValue({ id: 'bau_2' }); + + await service.createBrowserAutomation({ + taskId: 'tsk_1', + name: 'name', + targetUrl: 'https://example.com', + instruction: 'click', + }); + + expect(db.browserAutomation.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ scheduleFrequency: 'weekly' }), + }), + ); + }); + + it('sets one schedule for every automation on the task', async () => { + (db.browserAutomation.updateMany as jest.Mock).mockResolvedValue({ count: 3 }); + + const result = await service.setTaskSchedule('tsk_1', TaskFrequency.monthly); + + expect(db.browserAutomation.updateMany).toHaveBeenCalledWith({ + where: { taskId: 'tsk_1' }, + data: { scheduleFrequency: 'monthly' }, + }); + expect(result).toEqual({ + success: true, + scheduleFrequency: 'monthly', + updated: 3, + }); + }); + + it('stores explicit steps and mirrors the first onto the legacy columns', async () => { + (db.browserAutomation.create as jest.Mock).mockResolvedValue({ id: 'bau_1' }); + + await service.createBrowserAutomation({ + taskId: 't1', + name: 'A', + targetUrl: 'https://ignored.com', + instruction: 'ignored', + steps: [ + { + profileId: 'p1', + targetUrl: 'https://github.com', + instruction: 'screenshot 2fa', + evaluationCriteria: '2fa enforced', + }, + { targetUrl: 'https://aws.amazon.com', instruction: 'capture policy' }, + ], + }); + + const data = (db.browserAutomation.create as jest.Mock).mock.calls[0][0].data; + expect(data.targetUrl).toBe('https://github.com'); // mirrored from step 0 + expect(data.instruction).toBe('screenshot 2fa'); + expect(data.steps.create).toHaveLength(2); + expect(data.steps.create[0]).toMatchObject({ + order: 0, + profileId: 'p1', + targetUrl: 'https://github.com', + }); + expect(data.steps.create[1]).toMatchObject({ order: 1, profileId: null }); + }); + + it('wraps a single inline instruction as one step', async () => { + (db.browserAutomation.create as jest.Mock).mockResolvedValue({ id: 'bau_1' }); + + await service.createBrowserAutomation({ + taskId: 't1', + name: 'A', + targetUrl: 'https://x.com', + instruction: 'do it', + }); + + const data = (db.browserAutomation.create as jest.Mock).mock.calls[0][0].data; + expect(data.steps.create).toHaveLength(1); + expect(data.steps.create[0]).toMatchObject({ + order: 0, + targetUrl: 'https://x.com', + instruction: 'do it', + }); + }); + + it('replaces the step list when steps are supplied on update', async () => { + (db.browserAutomation.update as jest.Mock).mockResolvedValue({ id: 'bau_1' }); + + await service.updateBrowserAutomation('bau_1', { + steps: [{ targetUrl: 'https://okta.com', instruction: 'sso' }], + }); + + expect(db.browserAutomationStep.deleteMany).toHaveBeenCalledWith({ + where: { automationId: 'bau_1' }, + }); + const data = (db.browserAutomation.update as jest.Mock).mock.calls[0][0].data; + expect(data.targetUrl).toBe('https://okta.com'); + expect(data.steps.create).toHaveLength(1); + }); }); diff --git a/apps/api/src/browserbase/browserbase.service.ts b/apps/api/src/browserbase/browserbase.service.ts index 444b0e0fba..3b3d9f2e81 100644 --- a/apps/api/src/browserbase/browserbase.service.ts +++ b/apps/api/src/browserbase/browserbase.service.ts @@ -1,9 +1,18 @@ import { Injectable } from '@nestjs/common'; import { TaskFrequency } from '@db'; -import { BrowserAutomationCrudService } from './browser-automation-crud.service'; +import { tasks } from '@trigger.dev/sdk'; +import { + BrowserAutomationCrudService, + type BrowserAutomationStepInput, +} from './browser-automation-crud.service'; +import { BrowserAutomationDraftService } from './browser-automation-draft.service'; import { BrowserAutomationExecutionService } from './browser-automation-execution.service'; import { BrowserAuthProfileService } from './browser-auth-profile.service'; import { BrowserCredentialStorageService } from './browser-credential-storage.service'; +import type { + BrowserRunLivePhase, + EvidenceTimelineStep, +} from './browser-evidence-step-timeline'; import { BrowserEvidenceRunnerService } from './browser-evidence-runner.service'; import { BrowserbaseScreenshotService } from './browserbase-screenshot.service'; import { BrowserbaseSessionService } from './browserbase-session.service'; @@ -24,11 +33,73 @@ export class BrowserbaseService { private readonly automationCrud: BrowserAutomationCrudService = new BrowserAutomationCrudService( screenshots, ), - private readonly automationExecution: BrowserAutomationExecutionService = - new BrowserAutomationExecutionService(sessions, profiles, runner), + private readonly automationExecution: BrowserAutomationExecutionService = new BrowserAutomationExecutionService( + sessions, + profiles, + runner, + ), private readonly credentialStorage: BrowserCredentialStorageService = new BrowserCredentialStorageService(), ) {} + // Drafts have no dependencies; a field avoids Nest trying to DI-resolve it. + private readonly automationDrafts = new BrowserAutomationDraftService(); + + /** + * Kicks off login analysis as a background Trigger.dev run (browser + AI, which + * can outlast an HTTP/browser timeout) and returns a handle the client + * subscribes to for the result. + */ + async analyzeLogin(url: string): Promise<{ + runId: string; + publicAccessToken: string; + }> { + const handle = await tasks.trigger('analyze-vendor-login', { url }); + return { runId: handle.id, publicAccessToken: handle.publicAccessToken }; + } + + /** + * Kicks off a live test of an instruction the user hasn't saved yet. Creates + * the session up front (so the client shows it as a live view), then runs the + * instruction as a background Trigger.dev task that streams its steps. Nothing + * is persisted — this only proves the instruction out before it's saved. + */ + async testInstruction(input: { + organizationId: string; + taskId?: string; + profileId?: string; + targetUrl: string; + instruction: string; + evaluationCriteria?: string; + }): Promise<{ + runId: string; + publicAccessToken: string; + sessionId: string; + liveViewUrl: string; + }> { + const profile = await this.profiles.resolveProfileForTarget({ + organizationId: input.organizationId, + targetUrl: input.targetUrl, + profileId: input.profileId, + }); + const { sessionId, liveViewUrl } = + await this.createSessionWithContext(profile.contextId); + const handle = await tasks.trigger('test-vendor-instruction', { + organizationId: input.organizationId, + taskId: input.taskId, + profileId: profile.id, + targetUrl: input.targetUrl, + instruction: input.instruction, + evaluationCriteria: input.evaluationCriteria, + sessionId, + }); + return { + runId: handle.id, + publicAccessToken: handle.publicAccessToken, + sessionId, + liveViewUrl, + }; + } + async listAuthProfiles(organizationId: string) { return this.profiles.listProfiles(organizationId); } @@ -69,16 +140,73 @@ export class BrowserbaseService { return this.profiles.markNeedsReauth(input); } + async updateAuthProfile(input: { + organizationId: string; + profileId: string; + displayName?: string; + url?: string; + }) { + return this.profiles.updateProfile(input); + } + + async deleteAuthProfile(input: { organizationId: string; profileId: string }) { + // Best-effort: remove the stored login from 1Password before dropping the + // profile, so we don't leave orphaned secrets behind. + const profile = await this.profiles.getProfile(input); + if (profile?.vaultExternalItemRef) { + await this.credentialStorage.deleteProfileCredentialItem(profile); + } + return this.profiles.deleteProfile(input); + } + async storeAuthProfileCredentials(input: { organizationId: string; profileId: string; username: string; password: string; totpSeed?: string; + extraFields?: { label: string; value: string }[]; + usernameLabel?: string; }) { return this.credentialStorage.storeProfileCredentials(input); } + /** + * Kicks off the connect flow's first automated sign-in. Creates the browser + * session up front (so the client can show it as a live view — the user + * watches the auto-fill and takes over in place if it can't finish), then runs + * the sign-in as a background Trigger.dev task on that session (browser + AI, + * which can outlast an HTTP/browser timeout). + */ + async signInAuthProfile(input: { + organizationId: string; + profileId: string; + url: string; + mode?: 'password' | 'sso'; + /** Vendor's identifier-field label, forwarded so the streamed step is truthful. */ + usernameLabel?: string; + }): Promise<{ + runId: string; + publicAccessToken: string; + sessionId: string; + liveViewUrl: string; + }> { + const { sessionId, liveViewUrl } = await this.profiles.startProfileSession({ + organizationId: input.organizationId, + profileId: input.profileId, + }); + const handle = await tasks.trigger('sign-in-vendor-profile', { + ...input, + sessionId, + }); + return { + runId: handle.id, + publicAccessToken: handle.publicAccessToken, + sessionId, + liveViewUrl, + }; + } + async getOrCreateOrgContext(organizationId: string) { return this.profiles.getOrCreateOrgContext(organizationId); } @@ -103,6 +231,59 @@ export class BrowserbaseService { return this.sessions.checkLoginStatus(sessionId, url); } + // ─── Tenant-safe wrappers for the raw session/context endpoints ──────────── + // Every raw session/context operation must confirm the target belongs to the + // caller's org before acting on it (the persisted context holds the org's + // authenticated vendor cookies) — otherwise it's a cross-tenant IDOR. + + async createSessionForOrg(organizationId: string, contextId: string) { + await this.profiles.assertContextOwnedByOrg({ organizationId, contextId }); + return this.sessions.createSessionWithContext(contextId); + } + + async closeSessionForOrg( + organizationId: string, + sessionId: string, + ): Promise { + // Tolerant: a session that can no longer be resolved is already gone, so + // closing is a no-op. A resolvable session that isn't the org's is rejected. + let contextId: string | undefined; + try { + contextId = await this.sessions.getSessionContextId(sessionId); + } catch { + return; + } + if (!contextId) return; + await this.profiles.assertContextOwnedByOrg({ organizationId, contextId }); + await this.sessions.closeSession(sessionId); + } + + async navigateToUrlForOrg( + organizationId: string, + sessionId: string, + url: string, + ) { + await this.profiles.assertSessionOwnedByOrg({ organizationId, sessionId }); + return this.sessions.navigateToUrl(sessionId, url); + } + + async checkLoginStatusForOrg( + organizationId: string, + sessionId: string, + url: string, + ) { + await this.profiles.assertSessionOwnedByOrg({ organizationId, sessionId }); + return this.sessions.checkLoginStatus(sessionId, url); + } + + /** Confirms a client-supplied session belongs to the caller's org. */ + async assertSessionOwnedByOrg( + organizationId: string, + sessionId: string, + ): Promise { + await this.profiles.assertSessionOwnedByOrg({ organizationId, sessionId }); + } + async createBrowserAutomation( data: { taskId: string; @@ -111,6 +292,7 @@ export class BrowserbaseService { targetUrl: string; instruction: string; evaluationCriteria?: string; + steps?: BrowserAutomationStepInput[]; scheduleFrequency?: TaskFrequency; }, organizationId?: string, @@ -119,11 +301,17 @@ export class BrowserbaseService { } async getBrowserAutomation(automationId: string, organizationId?: string) { - return this.automationCrud.getBrowserAutomation(automationId, organizationId); + return this.automationCrud.getBrowserAutomation( + automationId, + organizationId, + ); } async getBrowserAutomationsForTask(taskId: string, organizationId?: string) { - return this.automationCrud.getBrowserAutomationsForTask(taskId, organizationId); + return this.automationCrud.getBrowserAutomationsForTask( + taskId, + organizationId, + ); } async updateBrowserAutomation( @@ -135,6 +323,7 @@ export class BrowserbaseService { instruction?: string; evaluationCriteria?: string; isEnabled?: boolean; + steps?: BrowserAutomationStepInput[]; scheduleFrequency?: TaskFrequency; }, organizationId?: string, @@ -147,10 +336,53 @@ export class BrowserbaseService { } async deleteBrowserAutomation(automationId: string, organizationId?: string) { - return this.automationCrud.deleteBrowserAutomation(automationId, organizationId); + return this.automationCrud.deleteBrowserAutomation( + automationId, + organizationId, + ); + } + + async setTaskSchedule( + taskId: string, + scheduleFrequency: TaskFrequency, + organizationId?: string, + ) { + return this.automationCrud.setTaskSchedule( + taskId, + scheduleFrequency, + organizationId, + ); + } + + // ===== Drafts (in-progress, unsaved automations) ===== + + listAutomationDrafts(taskId: string, organizationId: string) { + return this.automationDrafts.listDraftsForTask(taskId, organizationId); + } + + createAutomationDraft( + data: { taskId: string; name?: string; steps: unknown; createdById?: string | null }, + organizationId: string, + ) { + return this.automationDrafts.createDraft(data, organizationId); + } + + updateAutomationDraft( + draftId: string, + data: { name?: string; steps?: unknown }, + organizationId: string, + ) { + return this.automationDrafts.updateDraft(draftId, data, organizationId); + } + + deleteAutomationDraft(draftId: string, organizationId: string) { + return this.automationDrafts.deleteDraft(draftId, organizationId); } - async startAutomationWithLiveView(automationId: string, organizationId: string) { + async startAutomationWithLiveView( + automationId: string, + organizationId: string, + ) { return this.automationExecution.startAutomationWithLiveView( automationId, organizationId, @@ -162,15 +394,53 @@ export class BrowserbaseService { runId: string, sessionId: string, organizationId: string, + onSteps?: (steps: EvidenceTimelineStep[]) => void, ) { return this.automationExecution.executeAutomationOnSession( automationId, runId, sessionId, organizationId, + onSteps, + ); + } + + /** Runs the FULL step sequence on a live session, streaming the timeline. */ + async executeAutomationLive( + automationId: string, + runId: string, + sessionId: string, + organizationId: string, + onSteps?: (steps: EvidenceTimelineStep[]) => void, + onLiveView?: (url: string) => void, + onLivePhase?: (phase: BrowserRunLivePhase) => void, + ) { + return this.automationExecution.executeAutomationLive( + automationId, + runId, + sessionId, + organizationId, + onSteps, + onLiveView, + onLivePhase, ); } + /** + * Kick off the interactive Run as a background task so the live view can + * stream the AI's steps (same realtime mechanism as the Test flow). Returns a + * handle the composer subscribes to for `runSteps` + the final result. + */ + async startLiveAutomationExecution(input: { + automationId: string; + runId: string; + sessionId: string; + organizationId: string; + }): Promise<{ runId: string; publicAccessToken: string }> { + const handle = await tasks.trigger('execute-automation-live', input); + return { runId: handle.id, publicAccessToken: handle.publicAccessToken }; + } + async runBrowserAutomation(automationId: string, organizationId: string) { return this.automationExecution.runBrowserAutomation( automationId, @@ -190,7 +460,10 @@ export class BrowserbaseService { return this.automationCrud.getRunWithPresignedUrl(runId, organizationId); } - async getAutomationsWithPresignedUrls(taskId: string, organizationId?: string) { + async getAutomationsWithPresignedUrls( + taskId: string, + organizationId?: string, + ) { return this.automationCrud.getAutomationsWithPresignedUrls( taskId, organizationId, diff --git a/apps/api/src/browserbase/credential-vault.ts b/apps/api/src/browserbase/credential-vault.ts index aa9f111411..f87964f370 100644 --- a/apps/api/src/browserbase/credential-vault.ts +++ b/apps/api/src/browserbase/credential-vault.ts @@ -2,6 +2,8 @@ export interface RuntimeCredentialMaterial { username?: string; password?: string; totpCode?: string; + /** Extra site-specific fields (e.g. workspace, subdomain), filled by label. */ + extraFields?: { label: string; value: string }[]; } export const BROWSER_CREDENTIAL_VAULT_ADAPTER = @@ -16,9 +18,7 @@ export interface BrowserCredentialVaultAdapter { }): Promise; } -export class NoopBrowserCredentialVaultAdapter - implements BrowserCredentialVaultAdapter -{ +export class NoopBrowserCredentialVaultAdapter implements BrowserCredentialVaultAdapter { async resolveCredentialReference(_params: { profileId: string; provider?: string | null; diff --git a/apps/api/src/browserbase/dto/browserbase.dto.ts b/apps/api/src/browserbase/dto/browserbase.dto.ts index bf927c4b42..87ce68feec 100644 --- a/apps/api/src/browserbase/dto/browserbase.dto.ts +++ b/apps/api/src/browserbase/dto/browserbase.dto.ts @@ -1,11 +1,14 @@ import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +import { Type } from 'class-transformer'; import { + IsArray, IsEnum, IsNotEmpty, IsOptional, IsString, IsBoolean, IsUrl, + ValidateNested, } from 'class-validator'; import { TaskFrequency } from '@db'; import { IsSafeUrl } from '../validators/url-safety.validator'; @@ -19,6 +22,16 @@ export class CreateSessionDto { contextId: string; } +export class SetAuthProfileTotpDto { + @ApiProperty({ + description: + "The authenticator setup key (TOTP seed) shown once during the vendor's authenticator-app setup — not the rotating 6-digit code.", + }) + @IsString() + @IsNotEmpty() + totpSeed: string; +} + export class NavigateToUrlDto { @ApiProperty({ description: 'Browserbase session ID' }) @IsString() @@ -62,10 +75,61 @@ export class AuthStatusResponseDto { username?: string; } +// ===== Login analysis DTOs ===== + +export class AnalyzeLoginDto { + @ApiProperty({ description: 'Vendor sign-in URL to analyze' }) + @IsUrl({}, { message: 'url must be a valid URL' }) + @IsSafeUrl({ message: 'The provided URL is not allowed.' }) + @IsString() + @IsNotEmpty() + url: string; +} + +export class LoginRecommendationDto { + @ApiProperty({ enum: ['ready', 'works_with_checkins', 'manual'] }) + category: string; + + @ApiProperty() + headline: string; + + @ApiProperty() + detail: string; +} + +export class AnalyzeLoginResponseDto { + @ApiProperty({ + description: 'Trigger.dev run id for the background analysis', + }) + runId: string; + + @ApiProperty({ description: 'Public access token to subscribe to the run' }) + publicAccessToken: string; +} + +export class LoginAnalysisResponseDto { + @ApiProperty() + reachable: boolean; + + @ApiProperty({ type: [String] }) + detectedMethods: string[]; + + @ApiProperty({ enum: ['email', 'username', 'either', 'unknown'] }) + identifierType: string; + + @ApiProperty({ type: [Object] }) + extraFields: { label: string }[]; + + @ApiProperty({ type: () => LoginRecommendationDto }) + recommendation: LoginRecommendationDto; +} + // ===== Auth Profile DTOs ===== export class ResolveAuthProfileDto { - @ApiProperty({ description: 'Website URL to normalize into an auth profile hostname' }) + @ApiProperty({ + description: 'Website URL to normalize into an auth profile hostname', + }) @IsUrl({}, { message: 'url must be a valid URL' }) @IsSafeUrl({ message: 'The provided URL is not allowed.' }) @IsString() @@ -114,13 +178,92 @@ export class VerifyAuthProfileSessionDto { url: string; } +export class UpdateAuthProfileDto { + @ApiPropertyOptional({ description: 'Display name for the connection' }) + @IsString() + @IsOptional() + displayName?: string; + + @ApiPropertyOptional({ + description: + 'Sign-in URL. Changing to a different hostname signs the connection out.', + }) + @IsUrl({}, { message: 'url must be a valid URL' }) + @IsSafeUrl({ message: 'The provided URL is not allowed.' }) + @IsString() + @IsOptional() + url?: string; +} + export class MarkAuthProfileNeedsReauthDto { - @ApiPropertyOptional({ description: 'Reason the profile needs re-authentication' }) + @ApiPropertyOptional({ + description: 'Reason the profile needs re-authentication', + }) @IsString() @IsOptional() reason?: string; } +export class SignInAuthProfileDto { + @ApiProperty({ + description: 'Vendor URL to sign in to (the profile hostname).', + }) + @IsUrl({}, { message: 'url must be a valid URL' }) + @IsSafeUrl({ message: 'The provided URL is not allowed.' }) + @IsString() + @IsNotEmpty() + url: string; + + @ApiPropertyOptional({ + enum: ['password', 'sso'], + description: + "'password' fills stored credentials; 'sso' has the AI open the identity provider for the user to finish.", + }) + @IsEnum({ password: 'password', sso: 'sso' }) + @IsOptional() + mode?: 'password' | 'sso'; + + @ApiPropertyOptional({ + description: + "The vendor's detected identifier-field label (e.g. 'IAM username'), shown in the live sign-in steps instead of a generic 'username'.", + }) + @IsString() + @IsOptional() + usernameLabel?: string; +} + +export class SignInAuthProfileResponseDto { + @ApiProperty({ + description: 'Trigger.dev run id for the background automated sign-in', + }) + runId: string; + + @ApiProperty({ description: 'Public access token to subscribe to the run' }) + publicAccessToken: string; + + @ApiProperty({ + description: 'Browserbase session id the sign-in runs on (for take-over)', + }) + sessionId: string; + + @ApiProperty({ + description: 'Live view URL so the user can watch and take over the sign-in', + }) + liveViewUrl: string; +} + +export class CredentialExtraFieldDto { + @ApiProperty({ description: 'Field label as shown on the vendor login' }) + @IsString() + @IsNotEmpty() + label: string; + + @ApiProperty({ description: 'Field value' }) + @IsString() + @IsNotEmpty() + value: string; +} + export class StoreAuthProfileCredentialsDto { @ApiProperty({ description: 'Username or email for the vendor login' }) @IsString() @@ -139,6 +282,24 @@ export class StoreAuthProfileCredentialsDto { @IsString() @IsOptional() totpSeed?: string; + + @ApiPropertyOptional({ + type: [CredentialExtraFieldDto], + description: 'Extra site-specific fields (e.g. workspace, subdomain).', + }) + @IsArray() + @IsOptional() + @ValidateNested({ each: true }) + @Type(() => CredentialExtraFieldDto) + extraFields?: CredentialExtraFieldDto[]; + + @ApiPropertyOptional({ + description: + "The vendor's own label for the identifier field (e.g. \"IAM username\"), stored so sign-in steps and reconnects show the real field name.", + }) + @IsString() + @IsOptional() + usernameLabel?: string; } export class BrowserAuthProfileResponseDto { @@ -186,6 +347,11 @@ export class BrowserAuthProfileResponseDto { @ApiProperty() updatedAt: Date; + + @ApiPropertyOptional({ + description: 'Number of browser automations in the org that run on this connection', + }) + automationCount?: number; } export class ResolveAuthProfileResponseDto { @@ -206,6 +372,32 @@ export class VerifyAuthProfileResponseDto { // ===== Browser Automation DTOs ===== +export class BrowserAutomationStepDto { + @ApiPropertyOptional({ + description: 'Connection (browser auth profile) this step runs on', + }) + @IsString() + @IsOptional() + profileId?: string; + + @ApiProperty({ description: 'Starting URL for this step' }) + @IsUrl({}, { message: 'url must be a valid URL' }) + @IsSafeUrl({ message: 'The provided URL is not allowed.' }) + @IsString() + @IsNotEmpty() + targetUrl: string; + + @ApiProperty({ description: 'Natural language instruction for this step' }) + @IsString() + @IsNotEmpty() + instruction: string; + + @ApiPropertyOptional({ description: 'Optional pass/fail criterion for this step' }) + @IsString() + @IsOptional() + evaluationCriteria?: string; +} + export class CreateBrowserAutomationDto { @ApiProperty({ description: 'Task ID this automation belongs to' }) @IsString() @@ -242,6 +434,17 @@ export class CreateBrowserAutomationDto { @IsOptional() evaluationCriteria?: string; + @ApiPropertyOptional({ + type: [BrowserAutomationStepDto], + description: + 'Ordered steps for a multi-vendor automation. When provided, these define the run; the top-level targetUrl/instruction are taken from the first step.', + }) + @IsArray() + @ValidateNested({ each: true }) + @Type(() => BrowserAutomationStepDto) + @IsOptional() + steps?: BrowserAutomationStepDto[]; + @ApiPropertyOptional({ enum: TaskFrequency, description: 'Automation schedule cadence', @@ -251,6 +454,15 @@ export class CreateBrowserAutomationDto { scheduleFrequency?: TaskFrequency; } +export class SetTaskScheduleDto { + @ApiProperty({ + enum: TaskFrequency, + description: 'Cadence applied to every browser automation on the task', + }) + @IsEnum(TaskFrequency) + scheduleFrequency: TaskFrequency; +} + export class UpdateBrowserAutomationDto { @ApiPropertyOptional({ description: 'Automation name' }) @IsString() @@ -287,6 +499,17 @@ export class UpdateBrowserAutomationDto { @IsOptional() isEnabled?: boolean; + @ApiPropertyOptional({ + type: [BrowserAutomationStepDto], + description: + 'Ordered steps for a multi-vendor automation. When provided, they replace the automation’s existing steps.', + }) + @IsArray() + @ValidateNested({ each: true }) + @Type(() => BrowserAutomationStepDto) + @IsOptional() + steps?: BrowserAutomationStepDto[]; + @ApiPropertyOptional({ enum: TaskFrequency, description: 'Automation schedule cadence', @@ -296,6 +519,61 @@ export class UpdateBrowserAutomationDto { scheduleFrequency?: TaskFrequency; } +/** A draft step — everything optional, since a draft can be half-written. */ +export class DraftStepDto { + @ApiPropertyOptional() + @IsString() + @IsOptional() + profileId?: string; + + @ApiPropertyOptional() + @IsString() + @IsOptional() + targetUrl?: string; + + @ApiPropertyOptional() + @IsString() + @IsOptional() + instruction?: string; + + @ApiPropertyOptional() + @IsString() + @IsOptional() + evaluationCriteria?: string; +} + +export class CreateBrowserAutomationDraftDto { + @ApiProperty({ description: 'Task the draft belongs to' }) + @IsString() + @IsNotEmpty() + taskId: string; + + @ApiPropertyOptional({ description: 'Preview name, derived from the first step' }) + @IsString() + @IsOptional() + name?: string; + + @ApiProperty({ type: [DraftStepDto], description: 'The composer step state' }) + @IsArray() + @ValidateNested({ each: true }) + @Type(() => DraftStepDto) + steps: DraftStepDto[]; +} + +export class UpdateBrowserAutomationDraftDto { + @ApiPropertyOptional() + @IsString() + @IsOptional() + name?: string; + + @ApiPropertyOptional({ type: [DraftStepDto] }) + @IsArray() + @ValidateNested({ each: true }) + @Type(() => DraftStepDto) + @IsOptional() + steps?: DraftStepDto[]; +} + export class ExecuteAutomationSessionDto { @ApiProperty({ description: 'Browser automation run ID' }) @IsString() @@ -433,3 +711,51 @@ export class RunAutomationResponseDto { @ApiPropertyOptional() blockedReason?: string; } + +// ===== Instruction Test (coach loop) DTOs ===== + +export class TestInstructionDto { + @ApiProperty({ description: 'URL the AI should start from' }) + @IsUrl({}, { message: 'targetUrl must be a valid URL' }) + @IsSafeUrl({ message: 'The provided URL is not allowed.' }) + @IsString() + @IsNotEmpty() + targetUrl: string; + + @ApiProperty({ description: 'Natural language instruction to test' }) + @IsString() + @IsNotEmpty() + instruction: string; + + @ApiPropertyOptional({ + description: + 'Optional pass/fail criteria. When set, the test run gets a verdict.', + }) + @IsString() + @IsOptional() + evaluationCriteria?: string; + + @ApiPropertyOptional({ description: 'Connection (browser auth profile) to run under' }) + @IsString() + @IsOptional() + profileId?: string; + + @ApiPropertyOptional({ description: 'Task the instruction belongs to' }) + @IsString() + @IsOptional() + taskId?: string; +} + +export class TestInstructionResponseDto { + @ApiProperty({ description: 'Trigger.dev run id to subscribe to' }) + runId: string; + + @ApiProperty({ description: 'Public token for realtime subscription' }) + publicAccessToken: string; + + @ApiProperty({ description: 'Browserbase session id backing the live view' }) + sessionId: string; + + @ApiProperty({ description: 'Live view URL for watching the test run' }) + liveViewUrl: string; +} diff --git a/apps/api/src/browserbase/onepassword-credential-item.ts b/apps/api/src/browserbase/onepassword-credential-item.ts index 62a1c283ab..29a84746f6 100644 --- a/apps/api/src/browserbase/onepassword-credential-item.ts +++ b/apps/api/src/browserbase/onepassword-credential-item.ts @@ -18,6 +18,14 @@ export function buildItemReference(vaultId: string, itemId: string): string { return `op://${vaultId}/${itemId}`; } +export function parseItemReference(itemRef: string): { + vaultId?: string; + itemId?: string; +} { + const [vaultId, itemId] = itemRef.replace(/^op:\/\//, '').split('/'); + return { vaultId, itemId }; +} + export function buildFieldReference( itemRef: string, fieldTitle: string, diff --git a/apps/api/src/browserbase/onepassword-credential-vault.adapter.spec.ts b/apps/api/src/browserbase/onepassword-credential-vault.adapter.spec.ts index 3cac1a1171..3a1e843d7d 100644 --- a/apps/api/src/browserbase/onepassword-credential-vault.adapter.spec.ts +++ b/apps/api/src/browserbase/onepassword-credential-vault.adapter.spec.ts @@ -10,8 +10,16 @@ jest.mock('./onepassword-client', () => ({ const mockedGetClient = jest.mocked(getOnePasswordClient); -function clientWith(resolve: jest.Mock): OnePasswordClient { - return { secrets: { resolve } } as unknown as OnePasswordClient; +function clientWith( + resolve: jest.Mock, + itemsGet?: jest.Mock, +): OnePasswordClient { + return { + secrets: { resolve }, + items: { + get: itemsGet ?? jest.fn().mockRejectedValue(new Error('no items')), + }, + } as unknown as OnePasswordClient; } describe('OnePasswordCredentialVaultAdapter', () => { @@ -101,4 +109,33 @@ describe('OnePasswordCredentialVaultAdapter', () => { expect(result).toBeNull(); }); + + it('resolves extra custom fields, ignoring the reserved login fields', async () => { + const resolve = jest.fn((reference: string) => { + if (reference.endsWith('/username')) return 'alice'; + if (reference.endsWith('/password')) return 'pw'; + throw new Error('no totp'); + }); + const itemsGet = jest.fn().mockResolvedValue({ + fields: [ + { title: 'username', value: 'alice' }, + { title: 'password', value: 'pw' }, + { title: 'one-time password', value: 'seed' }, + { title: 'Workspace URL', value: 'acme.example.com' }, + { title: 'Empty', value: ' ' }, + ], + }); + mockedGetClient.mockResolvedValue(clientWith(resolve, itemsGet)); + + const result = await adapter.resolveCredentialReference({ + profileId: 'bap_1', + provider: '1password', + externalItemRef: 'op://vault123/item456', + }); + + expect(itemsGet).toHaveBeenCalledWith('vault123', 'item456'); + expect(result?.extraFields).toEqual([ + { label: 'Workspace URL', value: 'acme.example.com' }, + ]); + }); }); diff --git a/apps/api/src/browserbase/onepassword-credential-vault.adapter.ts b/apps/api/src/browserbase/onepassword-credential-vault.adapter.ts index 959ff1cab5..3e3324ea59 100644 --- a/apps/api/src/browserbase/onepassword-credential-vault.adapter.ts +++ b/apps/api/src/browserbase/onepassword-credential-vault.adapter.ts @@ -12,9 +12,21 @@ import { buildFieldReference, buildTotpReference, PASSWORD_FIELD_TITLE, + TOTP_FIELD_TITLE, USERNAME_FIELD_TITLE, } from './onepassword-credential-item'; +const RESERVED_FIELD_TITLES = new Set([ + USERNAME_FIELD_TITLE, + PASSWORD_FIELD_TITLE, + TOTP_FIELD_TITLE, +]); + +function parseItemRef(itemRef: string): { vaultId?: string; itemId?: string } { + const [vaultId, itemId] = itemRef.replace(/^op:\/\//, '').split('/'); + return { vaultId, itemId }; +} + /** * Resolves a browser auth profile's stored login (username, password, live TOTP) * from 1Password at run time using its `op:///` reference. Secrets @@ -51,16 +63,43 @@ export class OnePasswordCredentialVaultAdapter implements BrowserCredentialVault client, buildTotpReference(itemRef), ); + const extraFields = await this.resolveExtraFields(client, itemRef); - if (!username && !password && !totpCode) return null; + if (!username && !password && !totpCode && extraFields.length === 0) { + return null; + } return { username: username ?? undefined, password: password ?? undefined, totpCode: totpCode ?? undefined, + extraFields: extraFields.length > 0 ? extraFields : undefined, }; } + // Reads any site-specific custom fields (workspace, subdomain, …) the customer + // added. Best-effort: a vault without custom fields (or an older SDK) just + // yields none, and never blocks the standard username/password/TOTP path. + private async resolveExtraFields( + client: OnePasswordClient, + itemRef: string, + ): Promise<{ label: string; value: string }[]> { + try { + const { vaultId, itemId } = parseItemRef(itemRef); + if (!vaultId || !itemId) return []; + const item = await client.items.get(vaultId, itemId); + return item.fields + .filter( + (field) => + !RESERVED_FIELD_TITLES.has(field.title) && + Boolean(field.value?.trim()), + ) + .map((field) => ({ label: field.title, value: field.value })); + } catch { + return []; + } + } + private async resolveField( client: OnePasswordClient, reference: string, diff --git a/apps/api/src/browserbase/select-live-view-url.spec.ts b/apps/api/src/browserbase/select-live-view-url.spec.ts new file mode 100644 index 0000000000..75cab0e7c5 --- /dev/null +++ b/apps/api/src/browserbase/select-live-view-url.spec.ts @@ -0,0 +1,34 @@ +import { selectLiveViewUrl } from './browserbase-session.service'; + +const page = (url: string, tag: string) => ({ + url, + debuggerFullscreenUrl: `live://${tag}`, +}); + +describe('selectLiveViewUrl', () => { + const home = page('https://aws.amazon.com/', 'home'); + const signin = page('https://us-east-2.signin.aws.amazon.com/oauth', 'signin'); + const debug = { debuggerFullscreenUrl: 'live://session', pages: [home, signin] }; + + it('follows the tab matching the AI page by exact URL', () => { + expect(selectLiveViewUrl(debug, 'https://us-east-2.signin.aws.amazon.com/oauth')).toBe( + 'live://signin', + ); + }); + + it('matches by hostname when the exact URL has drifted (redirects)', () => { + expect( + selectLiveViewUrl(debug, 'https://us-east-2.signin.aws.amazon.com/console?x=1'), + ).toBe('live://signin'); + }); + + it('falls back to the newest tab when nothing matches', () => { + expect(selectLiveViewUrl(debug, 'https://unrelated.example.com/')).toBe('live://signin'); + }); + + it('uses the session-level view when there are no pages', () => { + expect(selectLiveViewUrl({ debuggerFullscreenUrl: 'live://session', pages: [] })).toBe( + 'live://session', + ); + }); +}); diff --git a/apps/api/src/trigger/browser-automation/analyze-vendor-login.ts b/apps/api/src/trigger/browser-automation/analyze-vendor-login.ts new file mode 100644 index 0000000000..fd47c1737c --- /dev/null +++ b/apps/api/src/trigger/browser-automation/analyze-vendor-login.ts @@ -0,0 +1,24 @@ +import { task } from '@trigger.dev/sdk'; +import { BrowserLoginAnalyzerService } from '../../browserbase/browser-login-analyzer.service'; +import type { LoginAnalysis } from '../../browserbase/browser-login-analysis'; + +const analyzer = new BrowserLoginAnalyzerService(); + +/** + * Runs the vendor login analysis (open a cloud browser, navigate to the sign-in + * page, and detect the login methods) in the background. Kept off the HTTP + * request path because the browser + AI work can outlast request/browser + * timeouts. The connect flow subscribes to the run for the result. + */ +export const analyzeVendorLogin = task({ + id: 'analyze-vendor-login', + // Browser open + AI navigate + AI extract; generous headroom, still well under + // the Browserbase session lifetime. + maxDuration: 180, + // Analysis already degrades to a manual fallback on failure, so a blind retry + // just wastes a browser session. + retry: { maxAttempts: 1 }, + run: async (payload: { url: string }): Promise => { + return analyzer.analyzeLogin(payload.url); + }, +}); diff --git a/apps/api/src/trigger/browser-automation/execute-automation-live.ts b/apps/api/src/trigger/browser-automation/execute-automation-live.ts new file mode 100644 index 0000000000..dd08c7c8e2 --- /dev/null +++ b/apps/api/src/trigger/browser-automation/execute-automation-live.ts @@ -0,0 +1,52 @@ +import { metadata, task } from '@trigger.dev/sdk'; +import { BrowserbaseService } from '../../browserbase/browserbase.service'; + +const browserbaseService = new BrowserbaseService(); + +/** + * Runs a saved automation on an already-started live session (from start-live) + * so the user can watch the AI work AND see its steps — the same live timeline + * the Test/connect flow shows. Kept off the HTTP request path because the + * browser + AI work can outlast request timeouts; the Run view subscribes to + * this run for live steps (`runSteps`) and the final result. + */ +export const executeAutomationLive = task({ + id: 'execute-automation-live', + // Matches the scheduled run + test caps: a multi-step task on a complex site + // can take a while; give it a safe budget instead of being cut off mid-run. + maxDuration: 60 * 10, + // A browser + AI run isn't safe to blindly retry (the session may be gone). + retry: { maxAttempts: 1 }, + run: async (payload: { + automationId: string; + runId: string; + sessionId: string; + organizationId: string; + }) => { + // Runs EVERY step (all vendors), not just the first — step 0 on the live + // session, later vendors in their own sessions. + return browserbaseService.executeAutomationLive( + payload.automationId, + payload.runId, + payload.sessionId, + payload.organizationId, + // Live activity timeline — surfaced to the Run view via realtime metadata. + // Steps are plain JSON; cast to the SDK's value type (named interfaces lack + // the index signature DeserializedJson structurally requires). + (steps) => + metadata.set( + 'runSteps', + steps as unknown as Parameters[1], + ), + // Each vendor's live view, so the Run iframe follows the run across vendors. + // A fresh view means we're live again — clear any transition state. + (liveViewUrl) => { + metadata.set('liveViewUrl', liveViewUrl); + metadata.set('livePhase', 'running'); + }, + // Live-view phase: 'switching' between vendors, 'finishing' at the end, so + // the Run view covers the (soon-to-disconnect) iframe with a calm overlay. + (livePhase) => metadata.set('livePhase', livePhase), + ); + }, +}); diff --git a/apps/api/src/trigger/browser-automation/run-browser-automation.spec.ts b/apps/api/src/trigger/browser-automation/run-browser-automation.spec.ts index d445fa8497..907d4b0788 100644 --- a/apps/api/src/trigger/browser-automation/run-browser-automation.spec.ts +++ b/apps/api/src/trigger/browser-automation/run-browser-automation.spec.ts @@ -22,7 +22,10 @@ jest.mock('@trycompai/email', () => ({ isUserUnsubscribed: jest.fn().mockResolvedValue(false), })); -import { shouldMarkTaskDoneAfterBrowserRun } from './run-browser-automation'; +import { + shouldMarkTaskDoneAfterBrowserRun, + shouldMarkTaskFailedAfterBrowserRun, +} from './run-browser-automation'; describe('shouldMarkTaskDoneAfterBrowserRun', () => { it('allows screenshot-only automations to complete tasks', () => { @@ -52,3 +55,23 @@ describe('shouldMarkTaskDoneAfterBrowserRun', () => { ).toBe(false); }); }); + +describe('shouldMarkTaskFailedAfterBrowserRun', () => { + it('fails the task when the control regressed (verdict fail)', () => { + expect(shouldMarkTaskFailedAfterBrowserRun({ evaluationStatus: 'fail' })).toBe(true); + }); + + it('fails the task when the connection needs reconnect', () => { + expect(shouldMarkTaskFailedAfterBrowserRun({ needsReauth: true })).toBe(true); + }); + + it('leaves the task alone on an infra-only failure (could not verify)', () => { + // No verdict and not a reauth issue → timeout / model unavailable / etc. + expect(shouldMarkTaskFailedAfterBrowserRun({})).toBe(false); + expect(shouldMarkTaskFailedAfterBrowserRun({ needsReauth: false })).toBe(false); + }); + + it('does not fail the task on a passing verdict', () => { + expect(shouldMarkTaskFailedAfterBrowserRun({ evaluationStatus: 'pass' })).toBe(false); + }); +}); diff --git a/apps/api/src/trigger/browser-automation/run-browser-automation.ts b/apps/api/src/trigger/browser-automation/run-browser-automation.ts index 35373e040e..9ae79d9f51 100644 --- a/apps/api/src/trigger/browser-automation/run-browser-automation.ts +++ b/apps/api/src/trigger/browser-automation/run-browser-automation.ts @@ -1,10 +1,6 @@ import { db } from '@db'; -import { orgParticipantMemberWhereForFlag } from '../../utils/org-participation'; import { logger, tags, task } from '@trigger.dev/sdk'; import { BrowserbaseService } from '../../browserbase/browserbase.service'; -import { triggerEmail } from '../../email/trigger-email'; -import { TaskStatusChangedEmail } from '../../email/templates/task-status-changed'; -import { isUserUnsubscribed } from '@trycompai/email'; const browserbaseService = new BrowserbaseService(); @@ -28,169 +24,26 @@ export function shouldMarkTaskDoneAfterBrowserRun(input: { } /** - * Send email notifications for task status change + * Whether a non-passing run means the task is genuinely no longer satisfied, so + * we should flip it to `failed`. A `fail` verdict = the control regressed; + * `needsReauth` = we can no longer sign in. An infra-only failure (timeout, + * model unavailable, …) has neither — that's "couldn't verify", not "control + * failed", so we leave the task alone and let it retry on the next tick. */ -async function sendTaskStatusChangeEmails(params: { - organizationId: string; - taskId: string; - taskTitle: string; - oldStatus: string; - newStatus: 'done' | 'failed'; -}) { - const { organizationId, taskId, taskTitle, oldStatus, newStatus } = params; - - try { - // Use the shared participation rule so this path stays aligned with the - // other task notifiers: internal (platform-operated) orgs include platform - // admins; other orgs exclude them. - const organization = await db.organization.findUnique({ - where: { id: organizationId }, - select: { name: true, isInternal: true }, - }); - const participantWhere = orgParticipantMemberWhereForFlag( - organization?.isInternal ?? false, - ); - const [task, allMembers] = await Promise.all([ - db.task.findUnique({ - where: { id: taskId }, - select: { - assignee: { - select: { - user: { - select: { - id: true, - name: true, - email: true, - }, - }, - }, - }, - }, - }), - db.member.findMany({ - where: { - organizationId, - deactivated: false, - ...participantWhere, - }, - select: { - role: true, - user: { - select: { - id: true, - name: true, - email: true, - }, - }, - }, - }), - ]); - - const organizationName = organization?.name ?? 'your organization'; - const appUrl = - process.env.NEXT_PUBLIC_APP_URL || - process.env.BETTER_AUTH_URL || - 'https://app.trycomp.ai'; - const taskUrl = `${appUrl}/${organizationId}/tasks/${taskId}`; - - // Filter for admins/owners - const adminMembers = allMembers.filter( - (member) => - member.role && - (member.role.includes('admin') || member.role.includes('owner')), - ); - - // Build recipient list: assignee + admins - const recipientMap = new Map< - string, - { id: string; name: string; email: string } - >(); - - // Add assignee - if (task?.assignee?.user?.id && task.assignee.user.email) { - recipientMap.set(task.assignee.user.id, { - id: task.assignee.user.id, - name: - task.assignee.user.name?.trim() || - task.assignee.user.email?.trim() || - 'User', - email: task.assignee.user.email, - }); - } - - // Add admin members - for (const member of adminMembers) { - if (member.user?.id && member.user.email) { - recipientMap.set(member.user.id, { - id: member.user.id, - name: member.user.name?.trim() || member.user.email?.trim() || 'User', - email: member.user.email, - }); - } - } - - const recipients = Array.from(recipientMap.values()); - - // Send emails to each recipient - await Promise.allSettled( - recipients.map(async (recipient) => { - // Check if user is unsubscribed - const isUnsubscribed = await isUserUnsubscribed( - db, - recipient.email, - 'taskAssignments', - organizationId, - ); - - if (isUnsubscribed) { - logger.info( - `Skipping notification: user ${recipient.email} is unsubscribed from task assignments`, - ); - return; - } - - try { - await triggerEmail({ - to: recipient.email, - subject: `Task "${taskTitle}" status changed to ${newStatus}`, - react: TaskStatusChangedEmail({ - toName: recipient.name, - toEmail: recipient.email, - taskTitle, - oldStatus: oldStatus.charAt(0).toUpperCase() + oldStatus.slice(1), - newStatus: newStatus === 'failed' ? 'Failed' : 'Done', - changedByName: 'Automation', - organizationName, - taskUrl, - }), - system: true, - }); - - logger.info(`Status change email sent to ${recipient.email}`); - } catch (error) { - logger.error( - `Failed to send status change email to ${recipient.email}`, - { - error: error instanceof Error ? error.message : 'Unknown error', - }, - ); - } - }), - ); - - logger.info( - `Sent ${recipients.length} status change notifications for task ${taskId} (status: ${newStatus})`, - ); - } catch (error) { - logger.error('Failed to send task status change emails', { - error: error instanceof Error ? error.message : 'Unknown error', - }); - } +export function shouldMarkTaskFailedAfterBrowserRun(input: { + evaluationStatus?: 'pass' | 'fail'; + needsReauth?: boolean; +}): boolean { + return input.evaluationStatus === 'fail' || input.needsReauth === true; } /** * Worker task that runs a single browser automation. - * Triggered by the orchestrator (browser-automations-schedule). + * + * Triggered by the per-org runner (run-org-browser-automations), which waits on + * a batch of these and sends ONE bundled failure email per org. This worker's + * job is only to run the automation and flip the task's status; notifications + * are the runner's responsibility (mirrors the integration check worker). */ export const runBrowserAutomation = task({ id: 'run-browser-automation', @@ -248,6 +101,10 @@ export const runBrowserAutomation = task({ organizationId, ); + // Whether THIS run flipped the task into `failed` (transition only). The + // per-org bundled failure email (next change) reports only these. + let statusChangedToFailed = false; + if (result.success) { logger.info(`Automation ${automationId} completed successfully`, { runId: result.runId, @@ -302,32 +159,36 @@ export const runBrowserAutomation = task({ evaluationStatus: result.evaluationStatus, }); - // Mark task as failed if auth issue - if (result.needsReauth) { - // Get current status before updating + // A real control failure (verdict `fail`) or a broken connection + // (`needsReauth`) means the task is no longer satisfied — flip it to + // `failed` so the dashboard reflects reality instead of keeping a stale + // `done`. An infra-only failure (timeout, model unavailable, …) is left + // alone (see shouldMarkTaskFailedAfterBrowserRun) so it retries next tick. + if ( + shouldMarkTaskFailedAfterBrowserRun({ + evaluationStatus: result.evaluationStatus, + needsReauth: result.needsReauth, + }) + ) { const taskBeforeUpdate = await db.task.findUnique({ where: { id: taskId }, select: { status: true }, }); const oldStatus = taskBeforeUpdate?.status ?? 'todo'; - await db.task.update({ - where: { id: taskId }, - data: { status: 'failed' }, - }); - - // Only send email notifications if status actually changed + // Transition only: don't re-flip / re-report a task that's already + // failed. The per-org runner (run-org-browser-automations) collects + // these transitions and sends one bundled failure email — covering both + // `needs_reauth` and control-regressed (`evaluation fail`) cases. if (oldStatus !== 'failed') { - await sendTaskStatusChangeEmails({ - organizationId, - taskId, - taskTitle, - oldStatus, - newStatus: 'failed', + await db.task.update({ + where: { id: taskId }, + data: { status: 'failed' }, }); + statusChangedToFailed = true; } else { logger.info( - `Skipping notification: task ${taskId} was already in failed status`, + `Task ${taskId} was already in failed status; not re-reporting`, ); } } @@ -357,6 +218,11 @@ export const runBrowserAutomation = task({ error: result.error, needsReauth: result.needsReauth, failureCode: result.failureCode, + // Consumed by the per-org bundled failure email (next change). + taskId, + taskTitle, + evaluationStatus: result.evaluationStatus, + statusChangedToFailed, }; }, }); diff --git a/apps/api/src/trigger/browser-automation/run-browser-automations-schedule.spec.ts b/apps/api/src/trigger/browser-automation/run-browser-automations-schedule.spec.ts index 6e09e8e537..5305ead112 100644 --- a/apps/api/src/trigger/browser-automation/run-browser-automations-schedule.spec.ts +++ b/apps/api/src/trigger/browser-automation/run-browser-automations-schedule.spec.ts @@ -1,6 +1,7 @@ import { TaskFrequency } from '@trycompai/db'; import { filterDueAutomations, + groupAutomationsByOrg, limitAutomationBatch, } from './run-browser-automations-schedule'; @@ -28,8 +29,11 @@ jest.mock('@trigger.dev/sdk', () => ({ }, })); -jest.mock('./run-browser-automation', () => ({ - runBrowserAutomation: { batchTrigger: jest.fn() }, +// The schedule now dispatches per-org runners; stub it so importing the +// orchestrator doesn't load the real runner (which evaluates queue()/task() and +// pulls in the integration email chain at module load). +jest.mock('./run-org-browser-automations', () => ({ + runOrgBrowserAutomations: { batchTrigger: jest.fn() }, })); const atUtc = (iso: string) => new Date(`${iso}T00:00:00.000Z`); @@ -214,3 +218,60 @@ describe('limitAutomationBatch', () => { expect(limited.map((automation) => automation.id)).toEqual(['ba_good']); }); }); + +describe('groupAutomationsByOrg', () => { + const automations = [ + { + id: 'ba_1', + name: 'GitHub login', + taskId: 'tsk_1', + task: { organizationId: 'org_1' }, + }, + { + id: 'ba_2', + name: 'GitLab login', + taskId: 'tsk_2', + task: { organizationId: 'org_1' }, + }, + { + id: 'ba_3', + name: 'Datadog login', + taskId: 'tsk_3', + task: { organizationId: 'org_2' }, + }, + ]; + + it('groups automations into one entry per org and attaches the org name', () => { + const groups = groupAutomationsByOrg({ + automations, + orgNameById: new Map([ + ['org_1', 'Acme'], + ['org_2', 'Globex'], + ]), + }); + + expect(groups).toHaveLength(2); + const org1 = groups.find((g) => g.organizationId === 'org_1'); + expect(org1?.organizationName).toBe('Acme'); + expect(org1?.automations.map((a) => a.automationId)).toEqual([ + 'ba_1', + 'ba_2', + ]); + expect(org1?.automations[0]).toEqual({ + automationId: 'ba_1', + automationName: 'GitHub login', + taskId: 'tsk_1', + }); + const org2 = groups.find((g) => g.organizationId === 'org_2'); + expect(org2?.automations.map((a) => a.automationId)).toEqual(['ba_3']); + }); + + it('falls back to a generic org name when one is missing', () => { + const groups = groupAutomationsByOrg({ + automations: [automations[0]], + orgNameById: new Map(), + }); + + expect(groups[0]?.organizationName).toBe('your organization'); + }); +}); diff --git a/apps/api/src/trigger/browser-automation/run-browser-automations-schedule.ts b/apps/api/src/trigger/browser-automation/run-browser-automations-schedule.ts index f06b42d884..001f14141b 100644 --- a/apps/api/src/trigger/browser-automation/run-browser-automations-schedule.ts +++ b/apps/api/src/trigger/browser-automation/run-browser-automations-schedule.ts @@ -1,6 +1,9 @@ import { db, TaskFrequency } from '@db'; import { logger, schedules } from '@trigger.dev/sdk'; -import { runBrowserAutomation } from './run-browser-automation'; +import { + runOrgBrowserAutomations, + type OrgBrowserAutomation, +} from './run-org-browser-automations'; import { isDueToday } from '../shared/is-due-today'; import { normalizeHostnameFromUrl } from '../../browserbase/browserbase-url'; @@ -86,9 +89,60 @@ export function limitAutomationBatch< return selected; } +/** One org's worth of scheduled automations, ready for the per-org runner. */ +export interface OrgAutomationGroup { + organizationId: string; + organizationName: string; + automations: OrgBrowserAutomation[]; +} + +/** + * Group the flat, already-capped automation list into one entry per org, + * attaching each org's display name. Pure + exported for unit testing. This is + * what lets failures bundle into a single email per org (one runner → one email) + * instead of one independent run per automation. Mirrors groupTasksByOrg in the + * integration orchestrator. + */ +export function groupAutomationsByOrg< + T extends { + id: string; + name: string; + taskId: string; + task: { organizationId: string }; + }, +>({ + automations, + orgNameById, +}: { + automations: T[]; + orgNameById: Map; +}): OrgAutomationGroup[] { + const byOrg = new Map(); + for (const a of automations) { + const organizationId = a.task.organizationId; + let group = byOrg.get(organizationId); + if (!group) { + group = { + organizationId, + organizationName: + orgNameById.get(organizationId) ?? 'your organization', + automations: [], + }; + byOrg.set(organizationId, group); + } + group.automations.push({ + automationId: a.id, + automationName: a.name, + taskId: a.taskId, + }); + } + return Array.from(byOrg.values()); +} + /** * Daily scheduled task (orchestrator) that finds all enabled browser automations - * and triggers individual runs for each. + * and dispatches one per-org runner for each org, which runs that org's + * automations and sends a single bundled failure email. */ export const browserAutomationsSchedule = schedules.task({ id: 'browser-automations-schedule', @@ -169,46 +223,61 @@ export const browserAutomationsSchedule = schedules.task({ ); } - // Build payloads for batch triggering - const triggerPayloads = limitedAutomations.map((automation) => ({ - payload: { - automationId: automation.id, - automationName: automation.name, - organizationId: automation.task.organizationId, - taskId: automation.taskId, - }, - })); + // Group the capped list into one runner per org so this org's failures + // bundle into a single email (mirrors the integration orchestrator). + const orgIds = [ + ...new Set(limitedAutomations.map((a) => a.task.organizationId)), + ]; + const orgs = await db.organization.findMany({ + where: { id: { in: orgIds } }, + select: { id: true, name: true }, + }); + const orgNameById = new Map(orgs.map((o) => [o.id, o.name] as const)); + + const orgGroups = groupAutomationsByOrg({ + automations: limitedAutomations, + orgNameById, + }); - // Trigger in batches of 500 - const BATCH_SIZE = 500; - let totalTriggered = 0; + // Dispatch per-org runners (fire-and-forget). Each runner internally waits + // on its own automation runs and sends one bundled failure email. + const ORG_BATCH_SIZE = 100; + const triggerPayloads = orgGroups.map((g) => ({ payload: g })); + let orgsTriggered = 0; + let automationsTriggered = 0; try { - for (let i = 0; i < triggerPayloads.length; i += BATCH_SIZE) { - const batch = triggerPayloads.slice(i, i + BATCH_SIZE); - await runBrowserAutomation.batchTrigger(batch); - totalTriggered += batch.length; + for (let i = 0; i < triggerPayloads.length; i += ORG_BATCH_SIZE) { + const batch = triggerPayloads.slice(i, i + ORG_BATCH_SIZE); + await runOrgBrowserAutomations.batchTrigger(batch); + orgsTriggered += batch.length; + automationsTriggered += batch.reduce( + (n, p) => n + p.payload.automations.length, + 0, + ); logger.info( - `Triggered batch ${Math.floor(i / BATCH_SIZE) + 1}: ${batch.length} automations`, + `Triggered org batch ${Math.floor(i / ORG_BATCH_SIZE) + 1}: ${batch.length} org(s)`, ); } - logger.info(`Triggered ${totalTriggered} browser automation runs`); + logger.info( + `Triggered ${orgsTriggered} org runner(s) covering ${automationsTriggered} automation(s)`, + ); return { success: true, - automationsTriggered: totalTriggered, + automationsTriggered, }; } catch (error) { logger.error('Failed to trigger browser automations', { error: error instanceof Error ? error.message : String(error), - triggeredBeforeError: totalTriggered, + triggeredBeforeError: automationsTriggered, }); return { success: false, - automationsTriggered: totalTriggered, + automationsTriggered, error: error instanceof Error ? error.message : String(error), }; } diff --git a/apps/api/src/trigger/browser-automation/run-org-browser-automations.spec.ts b/apps/api/src/trigger/browser-automation/run-org-browser-automations.spec.ts new file mode 100644 index 0000000000..e57fd1ce00 --- /dev/null +++ b/apps/api/src/trigger/browser-automation/run-org-browser-automations.spec.ts @@ -0,0 +1,165 @@ +// Importing the runner evaluates queue()/task() at module load — stub them. +// runBrowserAutomation + the integration email path are only referenced inside +// the task body (not at load), but stub them so nothing real is pulled in. +jest.mock('@trigger.dev/sdk', () => ({ + logger: { info: jest.fn(), warn: jest.fn(), error: jest.fn() }, + queue: jest.fn(() => ({ name: 'q' })), + task: (config: unknown) => config, +})); +jest.mock('./run-browser-automation', () => ({ + runBrowserAutomation: { batchTriggerAndWait: jest.fn() }, +})); +jest.mock('../integration-platform/run-org-integration-checks', () => ({ + sendBundledFailureEmails: jest.fn(), +})); + +import { collectFailedBrowserTasks } from './run-org-browser-automations'; + +describe('collectFailedBrowserTasks', () => { + it('reports a task that freshly transitioned to failed (control regressed)', () => { + const failed = collectFailedBrowserTasks([ + { + ok: true, + output: { + success: false, + taskId: 'tsk_1', + taskTitle: 'GitHub branch protection', + evaluationStatus: 'fail', + statusChangedToFailed: true, + }, + }, + ]); + + expect(failed).toEqual([ + { + taskId: 'tsk_1', + taskTitle: 'GitHub branch protection', + failedCount: 1, + totalCount: 1, + }, + ]); + }); + + it('reports a task whose connection needs reconnect (needsReauth)', () => { + const failed = collectFailedBrowserTasks([ + { + ok: true, + output: { + success: false, + taskId: 'tsk_1', + taskTitle: 'Datadog evidence', + needsReauth: true, + statusChangedToFailed: true, + }, + }, + ]); + + expect(failed.map((f) => f.taskId)).toEqual(['tsk_1']); + }); + + it('aggregates multiple automations of the same task into ONE entry with an honest tally', () => { + const failed = collectFailedBrowserTasks([ + // Same task, two automations run this batch: one passed, one failed. + { + ok: true, + output: { + success: true, + taskId: 'tsk_1', + taskTitle: 'Task 1', + statusChangedToFailed: false, + }, + }, + { + ok: true, + output: { + success: false, + taskId: 'tsk_1', + taskTitle: 'Task 1', + evaluationStatus: 'fail', + statusChangedToFailed: true, + }, + }, + ]); + + expect(failed).toEqual([ + { taskId: 'tsk_1', taskTitle: 'Task 1', failedCount: 1, totalCount: 2 }, + ]); + }); + + it('does not double-count when concurrent automations both report the transition', () => { + // The task→failed flip races: both children can read oldStatus !== failed + // and each returns statusChangedToFailed=true. Must still be ONE entry. + const failed = collectFailedBrowserTasks([ + { + ok: true, + output: { + success: false, + taskId: 'tsk_1', + taskTitle: 'Task 1', + evaluationStatus: 'fail', + statusChangedToFailed: true, + }, + }, + { + ok: true, + output: { + success: false, + taskId: 'tsk_1', + taskTitle: 'Task 1', + evaluationStatus: 'fail', + statusChangedToFailed: true, + }, + }, + ]); + + expect(failed).toHaveLength(1); + expect(failed[0]).toEqual({ + taskId: 'tsk_1', + taskTitle: 'Task 1', + failedCount: 2, + totalCount: 2, + }); + }); + + it('drops runs that did not transition, errored, passed, or lack a taskId', () => { + const failed = collectFailedBrowserTasks([ + // already-failed (no fresh transition) → dropped + { + ok: true, + output: { + success: false, + taskId: 't_already', + taskTitle: 'Already failed', + evaluationStatus: 'fail', + statusChangedToFailed: false, + }, + }, + // infra-only failure that didn't flip the task → dropped + { + ok: true, + output: { + success: false, + taskId: 't_infra', + taskTitle: 'Timed out', + statusChangedToFailed: false, + }, + }, + // passing run → dropped + { + ok: true, + output: { + success: true, + taskId: 't_pass', + taskTitle: 'Passed', + statusChangedToFailed: false, + }, + }, + // crashed child → dropped + { ok: false }, + // not-found / disabled run (no taskId) → dropped + { ok: true, output: { success: false } }, + ]); + + expect(failed).toEqual([]); + }); +}); diff --git a/apps/api/src/trigger/browser-automation/run-org-browser-automations.ts b/apps/api/src/trigger/browser-automation/run-org-browser-automations.ts new file mode 100644 index 0000000000..96a111c2ab --- /dev/null +++ b/apps/api/src/trigger/browser-automation/run-org-browser-automations.ts @@ -0,0 +1,171 @@ +import { logger, queue, task } from '@trigger.dev/sdk'; +import { runBrowserAutomation } from './run-browser-automation'; +import { + sendBundledFailureEmails, + type FailedTaskSummary, +} from '../integration-platform/run-org-integration-checks'; + +/** One automation scheduled for an org, as handed down by the orchestrator. */ +export interface OrgBrowserAutomation { + automationId: string; + automationName: string; + taskId: string; +} + +// Bound how many per-org runners wait concurrently. The child automation runs +// are themselves globally capped by runBrowserAutomation's own queue, so only +// the PARENTS are bounded here (mirrors run-org-integration-checks). +const orgRunnerQueue = queue({ + name: 'browser-automations-org-runner', + concurrencyLimit: 50, +}); + +// Chunk defensively — the orchestrator already caps automations per org, so in +// practice every org fits in a single batch. +const CHILD_BATCH_SIZE = 100; + +/** The subset of a browser-automation run's output the collector reads. All + * optional so a not-found / disabled / errored run (which omits taskId) is + * handled without a cast. */ +interface BrowserRunOutput { + success?: boolean; + taskId?: string; + taskTitle?: string; + statusChangedToFailed?: boolean; + evaluationStatus?: 'pass' | 'fail'; + needsReauth?: boolean; +} + +/** + * Reduce a batchTriggerAndWait over runBrowserAutomation to the tasks that + * freshly transitioned into `failed`, one entry per task. A task can own several + * automations (Task.browserAutomations is 1-to-many) and they run concurrently, + * so results are aggregated by taskId: `totalCount` = automations that ran for + * the task this batch, `failedCount` = how many did not pass. Only tasks this + * batch actually flipped to `failed` (statusChangedToFailed) are reported — + * matching the integration behavior (an already-failed task isn't re-notified, + * and an infra-only failure that didn't flip the task is skipped). Pure + + * exported for unit testing. + */ +export function collectFailedBrowserTasks( + runs: Array<{ ok: boolean; output?: BrowserRunOutput }>, +): FailedTaskSummary[] { + const byTask = new Map< + string, + { + taskTitle: string; + failedCount: number; + totalCount: number; + transitioned: boolean; + } + >(); + + for (const run of runs) { + if (!run.ok || !run.output) continue; + const o = run.output; + // Not-found / disabled runs carry no taskId — nothing to report. + if (typeof o.taskId !== 'string') continue; + + const entry = byTask.get(o.taskId) ?? { + taskTitle: o.taskTitle ?? o.taskId, + failedCount: 0, + totalCount: 0, + transitioned: false, + }; + entry.totalCount += 1; + const didNotPass = + o.success !== true || + o.evaluationStatus === 'fail' || + o.needsReauth === true; + if (didNotPass) entry.failedCount += 1; + if (o.statusChangedToFailed === true) entry.transitioned = true; + byTask.set(o.taskId, entry); + } + + const failed: FailedTaskSummary[] = []; + for (const [taskId, e] of byTask) { + if (!e.transitioned) continue; + failed.push({ + taskId, + taskTitle: e.taskTitle, + // A transitioned task failed at least once; keep the tally honest and >=1. + failedCount: Math.max(1, e.failedCount), + totalCount: Math.max(e.totalCount, e.failedCount, 1), + }); + } + return failed; +} + +/** + * Per-org runner. The daily orchestrator dispatches ONE of these per org + * (fire-and-forget). It runs that org's due browser automations in parallel via + * batchTriggerAndWait, then sends a SINGLE bundled email listing every task that + * failed this run — reusing the exact integration failure-email path + * (sendBundledFailureEmails) so both products notify identically. Mirrors + * run-org-integration-checks. + */ +export const runOrgBrowserAutomations = task({ + id: 'run-org-browser-automations', + queue: orgRunnerQueue, + // maxDuration is max COMPUTE time in SECONDS; the suspended wait during + // batchTriggerAndWait is checkpointed and doesn't count against it. 1h matches + // the sibling integration runner. + maxDuration: 60 * 60, + run: async (payload: { + organizationId: string; + organizationName: string; + automations: OrgBrowserAutomation[]; + }) => { + const { organizationId, organizationName, automations } = payload; + + logger.info( + `Running browser automations for org ${organizationId} (${automations.length} automation(s))`, + ); + + if (automations.length === 0) { + return { + organizationId, + automationsRun: 0, + failedTasks: 0, + emailed: false, + }; + } + + const allRuns: Array<{ ok: boolean; output?: BrowserRunOutput }> = []; + for (let i = 0; i < automations.length; i += CHILD_BATCH_SIZE) { + const batch = automations.slice(i, i + CHILD_BATCH_SIZE); + const batchResult = await runBrowserAutomation.batchTriggerAndWait( + batch.map((a) => ({ + payload: { + automationId: a.automationId, + automationName: a.automationName, + organizationId, + taskId: a.taskId, + }, + })), + ); + for (const r of batchResult.runs) { + allRuns.push(r.ok ? { ok: true, output: r.output } : { ok: false }); + } + } + + const failedTasks = collectFailedBrowserTasks(allRuns); + + logger.info( + `Org ${organizationId}: ${failedTasks.length} task(s) transitioned to failed`, + ); + + await sendBundledFailureEmails({ + organizationId, + organizationName, + failedTasks, + }); + + return { + organizationId, + automationsRun: automations.length, + failedTasks: failedTasks.length, + emailed: failedTasks.length > 0, + }; + }, +}); diff --git a/apps/api/src/trigger/browser-automation/sign-in-vendor-profile.ts b/apps/api/src/trigger/browser-automation/sign-in-vendor-profile.ts new file mode 100644 index 0000000000..8ee0f497fc --- /dev/null +++ b/apps/api/src/trigger/browser-automation/sign-in-vendor-profile.ts @@ -0,0 +1,46 @@ +import { metadata, task } from '@trigger.dev/sdk'; +import { + BrowserCredentialSigninService, + type AutoSignInResult, +} from '../../browserbase/browser-credential-signin.service'; + +const signin = new BrowserCredentialSigninService(); + +/** + * Runs the connect flow's first automated sign-in in the background: opens a + * cloud browser, fills the credentials the user just stored, and verifies the + * session. Kept off the HTTP request path because the browser + AI work can + * outlast request/browser timeouts. The connect flow subscribes to the run and, + * if it can't complete unattended, hands the live browser to the user. + */ +export const signInVendorProfile = task({ + id: 'sign-in-vendor-profile', + maxDuration: 240, + // A browser + AI sign-in isn't safe to blindly retry (it would re-enter + // credentials on a session that may already be gone), so run it once. + retry: { maxAttempts: 1 }, + run: async (payload: { + organizationId: string; + profileId: string; + url: string; + sessionId: string; + mode?: 'password' | 'sso'; + /** Vendor's identifier-field label (e.g. "IAM username") for a truthful step. */ + usernameLabel?: string; + }): Promise => { + return signin.signInWithStoredCredentials({ + ...payload, + // Live activity timeline — surfaced to the connect flow via realtime metadata. + // Steps are plain JSON; cast to the SDK's value type (named interfaces lack + // the index signature DeserializedJson structurally requires). + onSteps: (steps) => + metadata.set( + 'signinSteps', + steps as unknown as Parameters[1], + ), + // The tab the AI is on — so the connect flow's iframe follows it across + // new tabs (e.g. AWS opening its sign-in in a new tab). + onLiveView: (liveViewUrl) => metadata.set('signinLiveViewUrl', liveViewUrl), + }); + }, +}); diff --git a/apps/api/src/trigger/browser-automation/test-vendor-instruction.ts b/apps/api/src/trigger/browser-automation/test-vendor-instruction.ts new file mode 100644 index 0000000000..65b0489ac4 --- /dev/null +++ b/apps/api/src/trigger/browser-automation/test-vendor-instruction.ts @@ -0,0 +1,48 @@ +import { metadata, task } from '@trigger.dev/sdk'; +import { + BrowserInstructionTestService, + type InstructionTestResult, +} from '../../browserbase/browser-instruction-test.service'; + +const tester = new BrowserInstructionTestService(); + +/** + * Runs a not-yet-saved instruction against the connection's live session so the + * user can watch it work before committing it to the schedule. Kept off the HTTP + * request path because the browser + AI work can outlast request timeouts; the + * composer subscribes to the run for live steps and the final result. + */ +export const testVendorInstruction = task({ + id: 'test-vendor-instruction', + // Sign-in can take ~40s before navigation even starts, so give the run a + // generous, safe budget to finish a multi-step task on a complex site instead + // of being cut off mid-way. Matches the scheduled run's 10-minute cap. + maxDuration: 60 * 10, + // A browser + AI run isn't safe to blindly retry (the session may be gone), + // so run it once. + retry: { maxAttempts: 1 }, + run: async (payload: { + organizationId: string; + taskId?: string; + profileId?: string; + targetUrl: string; + instruction: string; + evaluationCriteria?: string; + sessionId: string; + }): Promise => { + return tester.testInstructionOnSession({ + ...payload, + // Live activity timeline — surfaced to the composer via realtime metadata. + // Steps are plain JSON; cast to the SDK's value type (named interfaces lack + // the index signature DeserializedJson structurally requires). + onSteps: (steps) => + metadata.set( + 'testSteps', + steps as unknown as Parameters[1], + ), + // Follow the agent across tabs so the composer's live view tracks the + // page it's actually working on, not the stale starting tab. + onLiveView: (liveViewUrl) => metadata.set('testLiveViewUrl', liveViewUrl), + }); + }, +}); diff --git a/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.test.tsx b/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.test.tsx index be1e1bdca1..2b4b098625 100644 --- a/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.test.tsx +++ b/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.test.tsx @@ -1,106 +1,123 @@ -import { render, screen } from '@testing-library/react'; -import type { - ButtonHTMLAttributes, - HTMLAttributes, - InputHTMLAttributes, - LabelHTMLAttributes, - ReactNode, -} from 'react'; -import { beforeEach, describe, expect, it, vi } from 'vitest'; import { - setMockPermissions, ADMIN_PERMISSIONS, AUDITOR_PERMISSIONS, mockHasPermission, + setMockPermissions, } from '@/test-utils/mocks/permissions'; +import { fireEvent, render, screen } from '@testing-library/react'; +import type { ButtonHTMLAttributes, InputHTMLAttributes, ReactNode } from 'react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Connection } from './connection-format'; vi.mock('@/hooks/use-permissions', () => ({ - usePermissions: () => ({ - permissions: {}, - hasPermission: mockHasPermission, - }), + usePermissions: () => ({ permissions: {}, hasPermission: mockHasPermission }), })); vi.mock('@/lib/api-client', () => ({ + // Never resolves, so the passed initialProfiles stay in state for the assertion. apiClient: { get: vi.fn(() => new Promise(() => undefined)), post: vi.fn().mockResolvedValue({ data: {} }), + patch: vi.fn().mockResolvedValue({ data: {} }), + delete: vi.fn().mockResolvedValue({ data: {} }), }, })); vi.mock('@trycompai/design-system', () => ({ - Badge: ({ children }: { children?: ReactNode }) => {children}, Button: ({ children, iconLeft, - iconRight, - loading, ...props - }: ButtonHTMLAttributes & { - iconLeft?: ReactNode; - iconRight?: ReactNode; - loading?: boolean; - }) => ( - ), - Card: ({ children }: HTMLAttributes) =>
{children}
, - CardContent: ({ children }: HTMLAttributes) =>
{children}
, - CardDescription: ({ children }: HTMLAttributes) =>

{children}

, - CardHeader: ({ children }: HTMLAttributes) =>
{children}
, - CardTitle: ({ children }: HTMLAttributes) =>

{children}

, Input: (props: InputHTMLAttributes) => , - Label: ({ children, ...props }: LabelHTMLAttributes) => ( - + Spinner: () => , + Section: ({ + title, + description, + actions, + children, + }: { + title?: ReactNode; + description?: ReactNode; + actions?: ReactNode; + children?: ReactNode; + }) => ( +
+ {title &&

{title}

} + {description &&

{description}

} + {actions} + {children} +
), - Spinner: () => , })); vi.mock('@trycompai/design-system/icons', () => ({ - Globe: () => , - Renew: () => , - Screen: () => , + Add: () => , })); +// Sub-components are covered on their own; stub them to focus on the client. +vi.mock('./ConnectionsTable', () => ({ + ConnectionsTable: ({ connections }: { connections: Connection[] }) => ( +
{connections.length} rows
+ ), +})); +vi.mock('./ManageConnectionSheet', () => ({ ManageConnectionSheet: () => null })); +// The shared connect flow is covered by the task suite; stub it so we only test +// that the client renders it (and doesn't pull the whole task subsystem). +vi.mock( + '@/app/(app)/[orgId]/tasks/[taskId]/components/browser-automations/ConnectVendorLoginFlow', + () => ({ ConnectVendorLoginFlow: () =>
}), +); + import { BrowserConnectionClient } from './BrowserConnectionClient'; +const profile: Connection = { + id: 'bap_1', + hostname: 'github.com', + loginIdentity: 'ci-bot@acme.com', + displayName: 'GitHub', + status: 'verified', + vaultExternalItemRef: 'op://vault/item', +}; + describe('BrowserConnectionClient permission gating', () => { - beforeEach(() => { - vi.clearAllMocks(); - }); + beforeEach(() => vi.clearAllMocks()); - it('shows Connect Browser button when user has integration:create permission', () => { + it('shows "Connect a vendor" and the table when the user can create integrations', () => { setMockPermissions(ADMIN_PERMISSIONS); - render(); + render(); - const connectButton = screen.getByRole('button', { name: /connect browser/i }); - expect(connectButton).toBeInTheDocument(); - expect(connectButton).not.toBeDisabled(); + expect(screen.getByRole('button', { name: /connect a vendor/i })).toBeInTheDocument(); + expect(screen.getByTestId('table')).toHaveTextContent('1 rows'); }); - it('hides Connect Browser button when user lacks integration:create permission', () => { + it('hides "Connect a vendor" for a read-only user but still lists connections', () => { setMockPermissions(AUDITOR_PERMISSIONS); - render(); + render(); - expect(screen.queryByRole('button', { name: /connect browser/i })).not.toBeInTheDocument(); - expect(screen.queryByRole('button', { name: /open browser/i })).not.toBeInTheDocument(); + expect(screen.queryByRole('button', { name: /connect a vendor/i })).not.toBeInTheDocument(); + expect(screen.getByTestId('table')).toBeInTheDocument(); }); - it('hides Connect Browser button when user has no permissions', () => { - setMockPermissions({}); - render(); + it('shows an empty state when there are no connections', () => { + setMockPermissions(ADMIN_PERMISSIONS); + render(); - expect(screen.queryByRole('button', { name: /connect browser/i })).not.toBeInTheDocument(); - expect(screen.queryByRole('button', { name: /open browser/i })).not.toBeInTheDocument(); + expect(screen.getByText(/no connections yet/i)).toBeInTheDocument(); + expect(screen.queryByTestId('table')).not.toBeInTheDocument(); }); - it('still shows the URL input regardless of permissions', () => { - setMockPermissions({}); - render(); + it('opens the shared connect flow when "Connect a vendor" is clicked', () => { + setMockPermissions(ADMIN_PERMISSIONS); + render(); - expect(screen.getByLabelText('Website URL')).toBeInTheDocument(); + expect(screen.queryByTestId('connect-flow')).not.toBeInTheDocument(); + fireEvent.click(screen.getAllByRole('button', { name: /connect a vendor/i })[0]); + expect(screen.getByTestId('connect-flow')).toBeInTheDocument(); }); }); diff --git a/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.tsx b/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.tsx index 5064647673..6786182efd 100644 --- a/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.tsx +++ b/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.tsx @@ -1,284 +1,275 @@ 'use client'; +// Reuse the task flow's proven connect/reconnect experience here so org-level +// connections use the same reliable path (method detection + automated +// credential entry + a working live takeover) instead of a bespoke, flaky one. +import { ConnectVendorLoginFlow } from '@/app/(app)/[orgId]/tasks/[taskId]/components/browser-automations/ConnectVendorLoginFlow'; +import { clearConnectState } from '@/app/(app)/[orgId]/tasks/[taskId]/components/browser-automations/connect-flow-storage'; import { usePermissions } from '@/hooks/use-permissions'; import { apiClient } from '@/lib/api-client'; -import { - Badge, - Button, - Card, - CardContent, - CardDescription, - CardHeader, - CardTitle, - Input, - Label, - Spinner, -} from '@trycompai/design-system'; -import { Globe, Screen } from '@trycompai/design-system/icons'; +import { Button, Section } from '@trycompai/design-system'; +import { Add } from '@trycompai/design-system/icons'; import { useCallback, useEffect, useState } from 'react'; -import { BrowserConnectionInstructions } from './BrowserConnectionInstructions'; -import { BrowserConnectionLiveView } from './BrowserConnectionLiveView'; -import { - BrowserConnectionProfileList, - type BrowserConnectionProfile, -} from './BrowserConnectionProfileList'; +import { toast } from 'sonner'; +import { methodOf, type Connection } from './connection-format'; +import { ConnectionsTable } from './ConnectionsTable'; +import { ManageConnectionSheet } from './ManageConnectionSheet'; -interface ResolveProfileResponse { - profile: BrowserConnectionProfile & { contextId: string }; - isNew: boolean; -} - -interface SessionResponse { - sessionId: string; - liveViewUrl: string; +interface BrowserConnectionClientProps { + organizationId: string; + initialProfiles?: Connection[]; } -interface AuthStatusResponse { - isLoggedIn: boolean; - username?: string; -} +/** The connect/reconnect flow the page is currently showing (full-screen). */ +type ActiveFlow = { kind: 'connect' } | { kind: 'reconnect'; connection: Connection }; -interface VerifyProfileResponse { - profile: BrowserConnectionProfile; - auth: AuthStatusResponse; -} +// Stable resume-state key for the shared connect flow on this page. Cleared each +// time the flow opens so "Connect a vendor" always starts fresh (a settings page +// has no "resume where I left off" — that would trap the user on a stale step). +const CONNECT_FLOW_KEY = 'org-connections'; -type Status = 'idle' | 'loading' | 'session-active' | 'checking'; +/** + * Org-level browser connections manager. Lists every vendor login the org has, + * with health, and wires the shared connect flow for connect / reconnect plus + * the profile API for rename / change-login / remove. + */ +export function BrowserConnectionClient({ + organizationId: _organizationId, + initialProfiles = [], +}: BrowserConnectionClientProps) { + const { hasPermission } = usePermissions(); + const canConnect = hasPermission('integration', 'create'); + const canUpdate = hasPermission('integration', 'update'); + const canDelete = hasPermission('integration', 'delete'); -interface BrowserConnectionClientProps { - organizationId: string; -} + const [profiles, setProfiles] = useState(initialProfiles); + const [flow, setFlow] = useState(null); + const [busy, setBusy] = useState(false); -export function BrowserConnectionClient({ organizationId }: BrowserConnectionClientProps) { - const { hasPermission } = usePermissions(); - const canManageBrowser = hasPermission('integration', 'create'); - const [status, setStatus] = useState('idle'); - const [profileId, setProfileId] = useState(null); - const [profiles, setProfiles] = useState([]); - const [sessionId, setSessionId] = useState(null); - const [liveViewUrl, setLiveViewUrl] = useState(null); - const [urlToCheck, setUrlToCheck] = useState('https://github.com'); - const [authStatus, setAuthStatus] = useState(null); - const [error, setError] = useState(null); - const hasContext = profiles.some((profile) => profile.status === 'verified'); + const [manageConnection, setManageConnection] = useState(null); + const [manageOpen, setManageOpen] = useState(false); - const checkContextStatus = useCallback(async () => { - try { - const res = await apiClient.get('/v1/browserbase/profiles'); - if (res.data) { - setProfiles(res.data); - const verifiedProfile = res.data.find((profile) => profile.status === 'verified'); - const firstProfile = verifiedProfile ?? res.data[0]; - setProfileId(firstProfile?.id ?? null); - } - } catch { - // Ignore - } + const fetchProfiles = useCallback(async () => { + const res = await apiClient.get('/v1/browserbase/profiles'); + setProfiles(Array.isArray(res.data) ? res.data : []); }, []); useEffect(() => { - checkContextStatus(); - }, [checkContextStatus]); - - const handleStartSession = async () => { - let startedSessionId: string | null = null; - try { - setError(null); - setStatus('loading'); + void fetchProfiles(); + }, [fetchProfiles]); - const profileRes = await apiClient.post( - '/v1/browserbase/profiles/resolve', - { url: urlToCheck }, - ); - if (profileRes.error || !profileRes.data) { - throw new Error(profileRes.error || 'Failed to create auth profile'); - } - setProfileId(profileRes.data.profile.id); - setProfiles((currentProfiles) => { - const rest = currentProfiles.filter( - (profile) => profile.id !== profileRes.data?.profile.id, - ); - return profileRes.data ? [profileRes.data.profile, ...rest] : currentProfiles; - }); + // After the shared flow connects or reconnects, refresh the list and return. + const handleFlowDone = useCallback( + (message: string) => { + void fetchProfiles(); + setFlow(null); + toast.success(message); + }, + [fetchProfiles], + ); - const sessionRes = await apiClient.post( - `/v1/browserbase/profiles/${profileRes.data.profile.id}/session`, - {}, - ); - if (sessionRes.error || !sessionRes.data) { - throw new Error(sessionRes.error || 'Failed to create session'); - } - startedSessionId = sessionRes.data.sessionId; - setSessionId(startedSessionId); - setLiveViewUrl(sessionRes.data.liveViewUrl); + // Always start the shared flow from a clean slate on this page. + const openConnect = useCallback(() => { + clearConnectState(CONNECT_FLOW_KEY); + setFlow({ kind: 'connect' }); + }, []); - // Navigate to the URL - await apiClient.post('/v1/browserbase/navigate', { - sessionId: startedSessionId, - url: urlToCheck, - }); + const handleReconnect = useCallback((connection: Connection) => { + setManageOpen(false); + clearConnectState(CONNECT_FLOW_KEY); + setFlow({ kind: 'reconnect', connection }); + }, []); - setStatus('session-active'); - } catch (err) { - setError(err instanceof Error ? err.message : 'Failed to start session'); - setSessionId(null); - setLiveViewUrl(null); - setStatus('idle'); + const handleManage = useCallback((connection: Connection) => { + setManageConnection(connection); + setManageOpen(true); + }, []); - // If we created a session but navigation failed, close it to avoid orphaned sessions - if (startedSessionId) { - try { - await apiClient.post('/v1/browserbase/session/close', { sessionId: startedSessionId }); - } catch { - // Ignore cleanup errors (don't mask original error) - } + const handleRename = useCallback( + async (connection: Connection, name: string) => { + setBusy(true); + try { + await apiClient.patch(`/v1/browserbase/profiles/${connection.id}`, { + displayName: name, + }); + await fetchProfiles(); + setManageOpen(false); + toast.success('Connection renamed.'); + } catch (err) { + toast.error(err instanceof Error ? err.message : 'Could not rename.'); + } finally { + setBusy(false); } - } - }; - - const handleCheckAuth = async () => { - if (!sessionId || !profileId) return; - - try { - setError(null); - setStatus('checking'); + }, + [fetchProfiles], + ); - const res = await apiClient.post( - `/v1/browserbase/profiles/${profileId}/verify`, - { sessionId, url: urlToCheck }, - ); - if (res.error || !res.data) { - throw new Error(res.error || 'Failed to check auth'); + const handleChangeLogin = useCallback( + async (connection: Connection, creds: { username: string; password: string }) => { + setBusy(true); + try { + await apiClient.post(`/v1/browserbase/profiles/${connection.id}/credentials`, creds); + await fetchProfiles(); + setManageOpen(false); + toast.success('Login updated. Reconnect to verify it works.'); + } catch (err) { + toast.error(err instanceof Error ? err.message : 'Could not update the login.'); + } finally { + setBusy(false); } + }, + [fetchProfiles], + ); - setAuthStatus(res.data.auth); - setProfiles((currentProfiles) => { - const rest = currentProfiles.filter((profile) => profile.id !== res.data?.profile.id); - return res.data ? [res.data.profile, ...rest] : currentProfiles; - }); + const handleSetTotp = useCallback(async (connection: Connection, totpSeed: string) => { + setBusy(true); + try { + await apiClient.post(`/v1/browserbase/profiles/${connection.id}/totp`, { totpSeed }); + toast.success('Automatic 2FA is on. Scheduled runs generate the code for you.'); + } catch (err) { + toast.error(err instanceof Error ? err.message : 'Could not save the authenticator key.'); + } finally { + setBusy(false); + } + }, []); - // Close the session after checking - await apiClient.post('/v1/browserbase/session/close', { sessionId }); - setSessionId(null); - setLiveViewUrl(null); - setProfileId(null); - setStatus('idle'); + const handleClearTotp = useCallback(async (connection: Connection) => { + setBusy(true); + try { + await apiClient.delete(`/v1/browserbase/profiles/${connection.id}/totp`); + toast.success('Automatic 2FA turned off.'); } catch (err) { - setError(err instanceof Error ? err.message : 'Failed to check auth'); - setStatus('session-active'); + toast.error(err instanceof Error ? err.message : 'Could not turn off automatic 2FA.'); + } finally { + setBusy(false); } - }; + }, []); - const handleCloseSession = async () => { - if (sessionId) { + const handleRemove = useCallback( + async (connection: Connection) => { + setBusy(true); try { - await apiClient.post('/v1/browserbase/session/close', { sessionId }); - } catch { - // Ignore + await apiClient.delete(`/v1/browserbase/profiles/${connection.id}`); + await fetchProfiles(); + setManageOpen(false); + toast.success('Connection removed.'); + } catch (err) { + toast.error(err instanceof Error ? err.message : 'Could not remove.'); + } finally { + setBusy(false); } - } - setSessionId(null); - setLiveViewUrl(null); - setProfileId(null); - setStatus('idle'); - }; + }, + [fetchProfiles], + ); + + // Connect / reconnect take over the whole panel via the shared flow — the same + // one the evidence task uses, so behaviour is identical and reliable. + if (flow) { + const reconnect = + flow.kind === 'reconnect' + ? { + url: flow.connection.lastAuthCheckUrl || `https://${flow.connection.hostname}`, + mode: methodOf(flow.connection), + } + : undefined; + return ( + handleFlowDone('Connection added.')} + onReconnected={() => handleFlowDone('Connection reconnected.')} + onCancel={() => setFlow(null)} + /> + ); + } return ( -
- {/* Status Card */} - - -
-
-
- -
-
- Browser Session - - {hasContext - ? 'At least one browser auth profile is verified' - : 'No verified browser auth profile yet'} - -
-
- - {hasContext ? 'Connected' : 'Not Connected'} - +
}> + Connect a vendor + + ) : undefined + } + > +
+ {/* Orient anyone who lands here from Settings without knowing the feature: + what it does, that it's unattended, and where the automations live. */} +
+
+ How it works
- - - {error &&

{error}

} +
    +
  1. + 1. + + Connect a vendor login here — Comp AI signs in for you, including any + two-factor codes. + +
    2. +
  2. + 2. + + On a schedule it signs in, screenshots the required page as audit evidence, + and re-signs in on its own when a session expires. + +
    3. +
  3. + 3. + + You add the automations that use these logins inside an{' '} + evidence task, in its + “Browser evidence” section. + +
    4. +
+
- {status === 'idle' && ( -
-
- -
-
- setUrlToCheck(e.target.value)} - /> -
- {canManageBrowser && ( - - )} -
-

- Open a browser session to authenticate with websites. Your login session will be - saved and used for browser automations. -

+ {profiles.length === 0 ? ( +
+
+
No connections yet
+

+ Connect a vendor login so Comp AI can sign in and capture evidence for your + browser automations. +

+ {canConnect && ( +
+
- - {authStatus && ( -
-
-
- - {authStatus.isLoggedIn - ? `Logged in${authStatus.username ? ` as ${authStatus.username}` : ''}` - : 'Not logged in'} - -
-
- )} -
- )} - - {status === 'loading' && ( -
- - Starting browser session... -
- )} - - - - {(status === 'session-active' || status === 'checking') && liveViewUrl && ( - +
+ ) : ( + )} - - -
+ +
+
); } diff --git a/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionInstructions.tsx b/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionInstructions.tsx deleted file mode 100644 index fa9e4fbed4..0000000000 --- a/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionInstructions.tsx +++ /dev/null @@ -1,33 +0,0 @@ -'use client'; - -import { Card, CardContent, CardHeader, CardTitle } from '@trycompai/design-system'; - -export function BrowserConnectionInstructions() { - return ( - - - How it works - - -
    -
  1. - Create a service account - Use a - dedicated account for browser automations when possible. -
    2. -
  2. - Authenticate per site - Each hostname gets - its own auth profile and Browserbase context. -
    3. -
  3. - Complete 2FA manually - Live View supports - manual approvals; Comp does not store passwords or TOTP secrets. -
    4. -
  4. - Reconnect when needed - Expired or blocked - profiles are shown above and skipped by scheduled runs. -
    5. -
- - - ); -} diff --git a/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionLiveView.tsx b/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionLiveView.tsx deleted file mode 100644 index 43ffa900d4..0000000000 --- a/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionLiveView.tsx +++ /dev/null @@ -1,65 +0,0 @@ -'use client'; - -import { - Button, - Card, - CardContent, - CardDescription, - CardHeader, - CardTitle, -} from '@trycompai/design-system'; -import { Renew } from '@trycompai/design-system/icons'; - -export function BrowserConnectionLiveView({ - liveViewUrl, - isChecking, - canManageBrowser, - onCheckAuth, - onClose, -}: { - liveViewUrl: string; - isChecking: boolean; - canManageBrowser: boolean; - onCheckAuth: () => void; - onClose: () => void; -}) { - return ( - - -
-
- Browser Session - - Log in to websites below. Your session will be saved for automations. - -
-
- - -
-
- - -
-