From f3bb8892b0e8b704a17442bdeae8804d8efc739a Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 17 Jul 2026 12:07:49 -0700 Subject: [PATCH] feat(config): add embedding endpoint override Co-authored-by: Pedro Larroy --- CHANGELOG.md | 2 + docs/configuration.md | 21 +++++++- docs/reference.md | 2 + internal/cli/app.go | 31 +++++++++-- internal/cli/app_test.go | 96 ++++++++++++++++++++++++++++++++++ internal/config/config.go | 3 ++ internal/config/config_test.go | 28 ++++++++++ 7 files changed, 178 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2ca650b..3da4c4a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## 0.8.1 - Unreleased +- Route embedding and semantic-search requests through a configurable embedding-only OpenAI-compatible endpoint while preserving the shared endpoint fallback. Thanks @larroy. + ## 0.8.0 - 2026-07-17 ### Highlights diff --git a/docs/configuration.md b/docs/configuration.md index 552bdf9f..458cfb16 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -108,6 +108,7 @@ token_env = "CRAWL_REMOTE_TOKEN" | --- | --- | --- | | `summary_model` | `gpt-5.4` | OpenAI model used by `summarize` | | `embed_model` | `text-embedding-3-small` | OpenAI embedding model | +| `embed_base_url` | _(empty)_ | Embedding-only OpenAI-compatible endpoint; falls back to the shared OpenAI endpoint | | `embed_dimensions` | `1024` | Must match the model | | `embedding_basis` | `title_original` | Supported: `title_original`, `dedupe_text`, `llm_key_summary` | | `vector_backend` | `exact` | Semantic search backend: `exact` or optional `turbovec` via Python `turbovec`; turbovec requires dimensions divisible by 8 | @@ -143,6 +144,23 @@ before updating gitcrawl. | `GITCRAWL_VECTOR_BACKEND` | Override semantic vector backend (`exact` or `turbovec`) | | `GITCRAWL_OPENAI_RETRY_DISABLED` | Set to `1` to disable OpenAI retry/backoff | | `GITCRAWL_OPENAI_BASE_URL` / `OPENAI_BASE_URL` | Custom OpenAI endpoint (e.g., for a proxy) | +| `GITCRAWL_EMBED_BASE_URL` | Override the endpoint used only for embedding requests | + +Embedding requests resolve their endpoint in this order: `GITCRAWL_EMBED_BASE_URL`, +`embed_base_url`, `GITCRAWL_OPENAI_BASE_URL`, `OPENAI_BASE_URL`, then the OpenAI +default. This permits a local or provider-specific embedding service without +moving summary requests away from the shared endpoint: + +```toml +[openai] +embed_base_url = "http://127.0.0.1:8080/v1" +``` + +```bash +gitcrawl configure --embed-base-url http://127.0.0.1:8080/v1 +``` + +Pass an empty value to `--embed-base-url` to restore the shared endpoint. ### GitHub overrides @@ -176,6 +194,7 @@ Interactive-friendly config edits without opening the file: ```bash gitcrawl configure --summary-model gpt-5.4 gitcrawl configure --embed-model text-embedding-3-small +gitcrawl configure --embed-base-url http://127.0.0.1:8080/v1 gitcrawl configure --embedding-basis title_original gitcrawl configure --json ``` @@ -192,7 +211,7 @@ gitcrawl doctor --json # for scripts gitcrawl doctor --locks --json ``` -Reports config path and existence, database path, source/runtime SQLite health, portable-store Git status, last repair action, whether `GITHUB_TOKEN` and `OPENAI_API_KEY` are present (and whether they came from env vs. config), the active summary/embed models, the embedding basis, and counts of repositories, threads, open threads, clusters, plus the last sync timestamp. If the API call surface is unsupported (older Go, missing crypto), `api_supported: false` is reported so you can investigate. +Reports config path and existence, database path, source/runtime SQLite health, portable-store Git status, last repair action, whether `GITHUB_TOKEN` and `OPENAI_API_KEY` are present (and whether they came from env vs. config), the active summary/embed models, embedding base URL, embedding basis, and counts of repositories, threads, open threads, clusters, plus the last sync timestamp. If the API call surface is unsupported (older Go, missing crypto), `api_supported: false` is reported so you can investigate. JSON output also reports the active executable and available Go/build metadata under `runtime`, plus read-only schema compatibility under `db_schema` and `source_db_schema`. Portable stores additionally expose `runtime_db_schema` for the runtime mirror. Inspect `state`, `pending_migrations`, `pr_details`, and `next_steps` to distinguish a current store from a pending migration, a newer unsupported schema, or a damaged database. Schema inspection never applies migrations; run a write-path command with the intended binary only after reviewing the reported drift. When the runtime database cannot be opened or its status cannot be read, doctor still writes the JSON diagnostics and includes `runtime_open_error` or `runtime_status_error`, then exits nonzero. Add `--locks` for read-only SQLite lock diagnostics. The JSON payload includes the database, WAL, SHM, and rollback-journal paths and sizes, read-only open status, `pragma quick_check` status, and best-effort writer-process detection. `safe_read_only_inspection` reflects archive health even when another SQLite connection is open; `writer_activity` separately reports whether `lsof` found another process with writable access to the database. `appears_idle` becomes true only when the archive is healthy, process detection is available, and no writer candidate is found. On platforms without supported process inspection, `process_detection.available` is `false`, `writer_activity` is `unknown`, and `appears_idle` remains false instead of treating missing process data as proof that no writer exists. diff --git a/docs/reference.md b/docs/reference.md index 7d119128..2c7b4aa1 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -53,6 +53,7 @@ path with `--config ` or `GITCRAWL_CONFIG`. | `GITCRAWL_EMBED_MODEL` | `text-embedding-3-small` | OpenAI embedding model | | `GITCRAWL_OPENAI_RETRY_DISABLED` | _(off)_ | Set `1` to disable OpenAI retry/backoff | | `GITCRAWL_OPENAI_BASE_URL` / `OPENAI_BASE_URL` | OpenAI default | Custom OpenAI endpoint | +| `GITCRAWL_EMBED_BASE_URL` | Shared OpenAI endpoint | Embedding-only OpenAI-compatible endpoint | ### GitHub overrides @@ -71,6 +72,7 @@ path with `--config ` or `GITCRAWL_CONFIG`. | --- | --- | | `summary_model` | `gpt-5.4` | | `embed_model` | `text-embedding-3-small` | +| `embed_base_url` | _(empty; uses the shared OpenAI endpoint)_ | | `embed_dimensions` | `1024` | | `embedding_basis` | `title_original` | | `vector_backend` | `exact`; `turbovec` requires Python `turbovec` and dimensions divisible by 8 | diff --git a/internal/cli/app.go b/internal/cli/app.go index b3f9aa18..792603a0 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -286,9 +286,10 @@ func (a *App) runConfigure(args []string) error { fs.SetOutput(io.Discard) summaryModel := fs.String("summary-model", "", "summary model") embedModel := fs.String("embed-model", "", "embedding model") + embedBaseURLFlag := fs.String("embed-base-url", "", "custom endpoint for the embedding model") embeddingBasis := fs.String("embedding-basis", "", "embedding basis") jsonOut := fs.Bool("json", false, "write JSON output") - if err := fs.Parse(normalizeCommandArgs(args, map[string]bool{"summary-model": true, "embed-model": true, "embedding-basis": true})); err != nil { + if err := fs.Parse(normalizeCommandArgs(args, map[string]bool{"summary-model": true, "embed-model": true, "embed-base-url": true, "embedding-basis": true})); err != nil { return usageErr(err) } a.applyCommandJSON(*jsonOut) @@ -311,6 +312,10 @@ func (a *App) runConfigure(args []string) error { cfg.OpenAI.EmbedModel = strings.TrimSpace(*embedModel) updated = true } + if flagWasSet(fs, "embed-base-url") { + cfg.OpenAI.EmbedBaseURL = strings.TrimSpace(*embedBaseURLFlag) + updated = true + } if strings.TrimSpace(*embeddingBasis) != "" { cfg.EmbeddingBasis = strings.TrimSpace(*embeddingBasis) updated = true @@ -325,10 +330,19 @@ func (a *App) runConfigure(args []string) error { "updated": updated || !configExists, "summary_model": cfg.OpenAI.SummaryModel, "embed_model": cfg.OpenAI.EmbedModel, + "embed_base_url": embedBaseURL(cfg), "embedding_basis": cfg.EmbeddingBasis, }, true) } +func flagWasSet(fs *flag.FlagSet, name string) bool { + found := false + fs.Visit(func(value *flag.Flag) { + found = found || value.Name == name + }) + return found +} + type refreshResult struct { Repository string `json:"repository"` Selected map[string]bool `json:"selected"` @@ -681,7 +695,7 @@ func (a *App) semanticSearchDocuments(ctx context.Context, rt localRuntime, repo if token.Value == "" { return nil, fmt.Errorf("semantic search requires OpenAI API key: set %s", rt.Config.OpenAI.APIKeyEnv) } - client := openai.New(openai.Options{APIKey: token.Value, BaseURL: openAIBaseURL(), Dimensions: rt.Config.OpenAI.EmbedDimensions, Retry: embedRetryOverride()}) + client := openai.New(openai.Options{APIKey: token.Value, BaseURL: embedBaseURL(rt.Config), Dimensions: rt.Config.OpenAI.EmbedDimensions, Retry: embedRetryOverride()}) queryVectors, err := client.Embed(ctx, rt.Config.OpenAI.EmbedModel, []string{query}) if err != nil { return nil, err @@ -1308,7 +1322,7 @@ func (a *App) embedRepository(ctx context.Context, owner, repoName string, optio if batchSize <= 0 { batchSize = 64 } - client := openai.New(openai.Options{APIKey: token.Value, BaseURL: openAIBaseURL(), Dimensions: rt.Config.OpenAI.EmbedDimensions, Retry: embedRetryOverride()}) + client := openai.New(openai.Options{APIKey: token.Value, BaseURL: embedBaseURL(rt.Config), Dimensions: rt.Config.OpenAI.EmbedDimensions, Retry: embedRetryOverride()}) type pendingBatch struct { start, end int @@ -1505,6 +1519,13 @@ func openAIBaseURL() string { return strings.TrimSpace(os.Getenv("OPENAI_BASE_URL")) } +func embedBaseURL(cfg config.Config) string { + if value := strings.TrimSpace(cfg.OpenAI.EmbedBaseURL); value != "" { + return value + } + return openAIBaseURL() +} + func githubBaseURL() string { if value := strings.TrimSpace(os.Getenv("GITCRAWL_GITHUB_BASE_URL")); value != "" { return value @@ -3866,6 +3887,7 @@ func (a *App) runDoctor(ctx context.Context, args []string) error { "last_sync_at": formatOptionalTime(storeStatus.LastSyncAt), "summary_model": cfg.OpenAI.SummaryModel, "embed_model": cfg.OpenAI.EmbedModel, + "embed_base_url": embedBaseURL(cfg), "embedding_basis": cfg.EmbeddingBasis, "api_supported": false, } @@ -3968,6 +3990,7 @@ func (a *App) runRemoteDoctor(ctx context.Context, cfg config.Config, configExis "openai_key_source": openAIKey.Source, "summary_model": cfg.OpenAI.SummaryModel, "embed_model": cfg.OpenAI.EmbedModel, + "embed_base_url": embedBaseURL(cfg), "embedding_basis": cfg.EmbeddingBasis, "api_supported": false, } @@ -5109,7 +5132,7 @@ Usage: "configure": `gitcrawl configure updates model fields in the config. Usage: - gitcrawl configure [--summary-model name] [--embed-model name] [--embedding-basis title_original] [--json] + gitcrawl configure [--summary-model name] [--embed-model name] [--embed-base-url url] [--embedding-basis title_original] [--json] `, "doctor": `gitcrawl doctor checks config, token, and database readiness. diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 31cbbffa..b26e32b8 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -4442,6 +4442,13 @@ func TestAppOutputModesAndUsageBranches(t *testing.T) { if got := openAIBaseURL(); got != "https://gitcrawl-openai.example/v1" { t.Fatalf("openAIBaseURL override = %q", got) } + if got := embedBaseURL(config.Config{}); got != "https://gitcrawl-openai.example/v1" { + t.Fatalf("embedBaseURL fallback = %q", got) + } + embedCfg := config.Config{OpenAI: config.OpenAIConfig{EmbedBaseURL: " https://embed.example/v1 "}} + if got := embedBaseURL(embedCfg); got != "https://embed.example/v1" { + t.Fatalf("embedBaseURL override = %q", got) + } t.Setenv("GITHUB_BASE_URL", "https://github.example") if got := githubBaseURL(); got != "https://github.example" { t.Fatalf("githubBaseURL fallback = %q", got) @@ -4455,6 +4462,95 @@ func TestAppOutputModesAndUsageBranches(t *testing.T) { } } +func TestConfigureEmbedBaseURL(t *testing.T) { + ctx := context.Background() + configPath := filepath.Join(t.TempDir(), "config.toml") + var stdout bytes.Buffer + app := New() + app.Stdout = &stdout + if err := app.Run(ctx, []string{"--config", configPath, "configure", "--embed-base-url", "https://embed.example/v1", "--json"}); err != nil { + t.Fatalf("configure embed base URL: %v", err) + } + cfg, err := config.Load(configPath) + if err != nil { + t.Fatalf("load config: %v", err) + } + if cfg.OpenAI.EmbedBaseURL != "https://embed.example/v1" { + t.Fatalf("saved embed_base_url = %q", cfg.OpenAI.EmbedBaseURL) + } + if !strings.Contains(stdout.String(), `"embed_base_url": "https://embed.example/v1"`) { + t.Fatalf("configure output = %q", stdout.String()) + } + + if err := app.Run(ctx, []string{"--config", configPath, "configure", "--embed-base-url", ""}); err != nil { + t.Fatalf("clear embed base URL: %v", err) + } + cfg, err = config.Load(configPath) + if err != nil { + t.Fatalf("reload config: %v", err) + } + if cfg.OpenAI.EmbedBaseURL != "" { + t.Fatalf("cleared embed_base_url = %q", cfg.OpenAI.EmbedBaseURL) + } +} + +func TestEmbedUsesConfiguredBaseURL(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + configPath := filepath.Join(dir, "config.toml") + dbPath := filepath.Join(dir, "gitcrawl.db") + if err := New().Run(ctx, []string{"--config", configPath, "init", "--db", dbPath}); err != nil { + t.Fatalf("init: %v", err) + } + + seedCommandFlowStore(t, dbPath) + + var configuredCalls atomic.Int64 + configured := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + configuredCalls.Add(1) + if r.URL.Path != "/embeddings" { + http.Error(w, "unexpected path", http.StatusNotFound) + return + } + var request struct { + Input []string `json:"input"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil || len(request.Input) == 0 { + http.Error(w, "missing input", http.StatusBadRequest) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "data": []map[string]any{{"index": 0, "embedding": []float64{0.25, 0.75}}}, + }) + })) + defer configured.Close() + var sharedCalls atomic.Int64 + shared := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + sharedCalls.Add(1) + http.Error(w, "shared endpoint should not be used", http.StatusInternalServerError) + })) + defer shared.Close() + + configure := New() + if err := configure.Run(ctx, []string{"--config", configPath, "configure", "--embed-base-url", configured.URL}); err != nil { + t.Fatalf("configure: %v", err) + } + t.Setenv("OPENAI_API_KEY", "x") + t.Setenv("GITCRAWL_OPENAI_BASE_URL", shared.URL) + var stdout bytes.Buffer + embed := New() + embed.Stdout = &stdout + if err := embed.Run(ctx, []string{"--config", configPath, "embed", "openclaw/openclaw", "--limit", "1", "--json"}); err != nil { + t.Fatalf("embed: %v", err) + } + if configuredCalls.Load() != 1 || sharedCalls.Load() != 0 { + t.Fatalf("endpoint calls: configured=%d shared=%d", configuredCalls.Load(), sharedCalls.Load()) + } + if !strings.Contains(stdout.String(), `"embedded": 1`) { + t.Fatalf("embed output = %q", stdout.String()) + } +} + func TestGlobalCommandBranches(t *testing.T) { ctx := context.Background() cases := []struct { diff --git a/internal/config/config.go b/internal/config/config.go index 6d5087f9..c7355b15 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -39,6 +39,7 @@ type OpenAIConfig struct { APIKeyEnv string `toml:"api_key_env"` SummaryModel string `toml:"summary_model"` EmbedModel string `toml:"embed_model"` + EmbedBaseURL string `toml:"embed_base_url"` EmbedDimensions int `toml:"embed_dimensions"` BatchSize int `toml:"batch_size"` Concurrency int `toml:"concurrency"` @@ -178,6 +179,7 @@ func (c *Config) Normalize() error { if c.OpenAI.EmbedModel == "" { c.OpenAI.EmbedModel = def.OpenAI.EmbedModel } + c.OpenAI.EmbedBaseURL = strings.TrimSpace(c.OpenAI.EmbedBaseURL) if c.OpenAI.EmbedDimensions <= 0 { c.OpenAI.EmbedDimensions = def.OpenAI.EmbedDimensions } @@ -218,6 +220,7 @@ func (c *Config) Normalize() error { func (c *Config) ApplyRuntimeEnv() { c.OpenAI.SummaryModel = c.envOrDefault("GITCRAWL_SUMMARY_MODEL", c.OpenAI.SummaryModel) c.OpenAI.EmbedModel = c.envOrDefault("GITCRAWL_EMBED_MODEL", c.OpenAI.EmbedModel) + c.OpenAI.EmbedBaseURL = c.envOrDefault("GITCRAWL_EMBED_BASE_URL", c.OpenAI.EmbedBaseURL) c.VectorBackend = c.envOrDefault("GITCRAWL_VECTOR_BACKEND", c.VectorBackend) c.TUI.DefaultLayout = c.envOrDefault("GITCRAWL_TUI_LAYOUT", c.TUI.DefaultLayout) c.Remote.Mode = c.envOrDefault("GITCRAWL_REMOTE_MODE", c.Remote.Mode) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 2ebc827f..b07beba9 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -451,6 +451,34 @@ GITCRAWL_EMBED_MODEL = "embed-from-config-env" } } +func TestLoadRuntimeAppliesEmbedBaseURLOverride(t *testing.T) { + t.Setenv("GITCRAWL_EMBED_BASE_URL", "https://process.example.com/v1") + + path := filepath.Join(t.TempDir(), "config.toml") + if err := os.WriteFile(path, []byte(` +[openai] +embed_base_url = "https://config.example.com/v1" +`), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + + loaded, err := Load(path) + if err != nil { + t.Fatalf("load config: %v", err) + } + if loaded.OpenAI.EmbedBaseURL != "https://config.example.com/v1" { + t.Fatalf("config embed_base_url = %q", loaded.OpenAI.EmbedBaseURL) + } + + runtime, err := LoadRuntime(path) + if err != nil { + t.Fatalf("load runtime config: %v", err) + } + if runtime.OpenAI.EmbedBaseURL != "https://process.example.com/v1" { + t.Fatalf("runtime embed_base_url = %q", runtime.OpenAI.EmbedBaseURL) + } +} + func TestLoadDoesNotApplyRuntimeEnvFallback(t *testing.T) { t.Setenv("GITCRAWL_SUMMARY_MODEL", "summary-from-process") t.Setenv("GITCRAWL_EMBED_MODEL", "")