diff --git a/.agents/sow/current/SOW-0102-20260603-quality-complexity-duplication-coverage.md b/.agents/sow/current/SOW-0102-20260603-quality-complexity-duplication-coverage.md index 6376c1d..782850e 100644 --- a/.agents/sow/current/SOW-0102-20260603-quality-complexity-duplication-coverage.md +++ b/.agents/sow/current/SOW-0102-20260603-quality-complexity-duplication-coverage.md @@ -4,7 +4,7 @@ Status: in-progress -Sub-state: twenty-fourth implementation slice validated; entity detail selection PR pending +Sub-state: twenty-fifth implementation slice validated; ASN database processing PR pending ## Requirements @@ -2023,6 +2023,57 @@ Open decisions: - No new user design decision is required because the slice is behavior-preserving quality work under the previously approved quality plan. +## Slice 25 Results + +Changes made: + +- Moved ASN provider lifecycle details into `pkg/engine/asn_provider_processing.go`. +- Kept `processASNDatabases` responsible for ASN source discovery, runtime ASN directory creation, context cancellation checks, result map ownership, and close-on-error cleanup. +- Split the per-provider flow into focused helpers for: + - provider source metadata snapshots; + - ASN format validation; + - provider path construction and staged source selection; + - extraction decisions; + - database availability checks; + - ASN database open failures; + - loaded provider stat and stale-status recording. +- Added focused tests for no ASN sources, invalid and wrong-role formats, missing database files, successful existing TSV loads, staged gzip source loads after download failures, and fatal open failures after a previous provider loaded successfully. +- Preserved existing fatal/non-fatal provider behavior, staged source handling, stale load accounting, status strings, source metadata snapshots, and dataset cleanup ownership. + +Measured result: + +- Baseline: `pkg/engine/asn.go:39` `(*Engine).processASNDatabases` was 121 lines with complexity 22 and `63.3%` direct coverage. +- After refactor: no production function appears in `tools/archposture` large-function output. +- `processASNDatabases` direct coverage moved to `84.2%`. +- New helper coverage includes `processASNProvider` at `100.0%`, `(*asnProviderProcessor).process` at `92.9%`, `applySourceConfig` at `100.0%`, `validateFormat` at `100.0%`, `databaseAvailable` at `100.0%`, `openDatabase` at `100.0%`, and `recordLoaded` at `90.0%`. +- `pkg/engine` coverage moved from `71.7%` to `71.9%`. +- Root coverage by `go tool cover -func=coverage.out` remains `72.9%`. +- `tools/archposture` after this slice: source files `632`, source lines `127395`, large files `49`, large functions `25`, and production large functions `0`. +- Remaining production complexity target: none reported by local `tools/archposture`. + +Tests or equivalent validation: + +- `go test ./pkg/engine`: passed. +- `go test -coverprofile=/tmp/update-ipsets-engine-slice25.cover -covermode=atomic ./pkg/engine`: passed, `71.9%`. +- `go tool cover -func=/tmp/update-ipsets-engine-slice25.cover`: passed; `processASNDatabases` `84.2%`, package total `71.9%`. +- `go run ./tools/archposture -root . > /tmp/update-ipsets-archposture-slice25.json`: passed. +- `make lint`: passed. +- `make staticcheck`: passed. +- `make golangci-lint`: passed with `0 issues`. +- `CI=true make coverage`: passed, root total `72.9%`. +- `make test-strict`: passed. +- `git diff --check`: passed. +- Durable-artifact forbidden-name scan over added diff lines found no newly added personal name, authorship, tool, or vendor-attribution text. + +Artifact maintenance gate: + +- AGENTS.md: no update needed. +- Runtime project skills: no update needed; no new durable process rule was found. +- Specs: no update needed; ASN provider lifecycle semantics are unchanged. +- End-user/operator docs: no update needed. +- End-user/operator skills: no update needed. +- SOW lifecycle: remains in `.agents/sow/current/`; Slice 25 is validated and pending PR merge. + ## Slice 16 Results Changes made: @@ -3122,11 +3173,98 @@ Artifact maintenance gate: - Specs: no update needed; selected entity detail semantics are unchanged. - End-user/operator docs: no update needed. - End-user/operator skills: no update needed. -- SOW lifecycle: remains in `.agents/sow/current/`; Slice 24 is validated and pending PR merge. +- SOW lifecycle: remains in `.agents/sow/current/`; Slice 24 merged through PR #28 as merge commit `1350bc9ff4caea3dd9a72f282f3427db8c7d7180`. + +## Pre-Implementation Gate - Slice 25 + +Status: ready. + +Problem / root-cause model: + +- Facts: after the Slice 24 merge, local `tools/archposture` reports exactly one remaining production large function: `pkg/engine/asn.go:39` `(*Engine).processASNDatabases` at 121 lines with complexity 22. +- Facts: `go test -coverprofile=/tmp/update-ipsets-engine-slice25-baseline.cover -covermode=atomic ./pkg/engine` reports `pkg/engine` coverage at `71.7%`; `go tool cover` reports `processASNDatabases` at `63.3%`. +- Working theory: the function is large because it combines provider discovery, runtime directory setup, per-provider attempt lifecycle, source metadata snapshotting, format validation, staged archive selection, extraction, availability checks, ASN database opening, stats collection, freshness/stale accounting, logging, and cleanup. + +Evidence reviewed: + +- `pkg/engine/asn.go` +- `pkg/engine/geoloc.go` +- `pkg/engine/format_handlers.go` +- `pkg/engine/asn_formats.go` +- `pkg/asnloc/asnloc.go` +- `pkg/asnloc/loader_iptoasn_test.go` +- `pkg/engine/pipeline_integrity_scenario_test.go` +- `pkg/engine/engine_fixture_test.go` +- `/tmp/update-ipsets-archposture-main.json` +- `/tmp/update-ipsets-engine-slice25-baseline.cover` +- Project coding, testing, hygiene, Go best-practices, Go behavioral-testing, and content-surface skills. + +Affected contracts and surfaces: + +- ASN provider source metadata captured in cache entries. +- ASN format validation and wrong-role rejection. +- ASN provider directory layout under `runtime.LibDir/asn//`. +- Staged archive handling through `preferStagedPath`. +- Extraction policy for compressed ASN provider formats. +- Missing database, failed open, loaded, and stale-after-download-failure status transitions. +- ASN database handle ownership and close-on-error behavior. +- SOW only; no docs or specs are expected to change because the slice is behavior-preserving. + +Existing patterns to reuse: + +- `processGeoIPDatabases` provider lifecycle structure where applicable. +- Existing `cache.Entry` provider status transition APIs. +- Existing `newEngineFixture` for engine tests. +- Existing `asnloc.Open` parser behavior using small `iptoasn_combined_tsv` fixtures. +- Existing generated file modes and temporary directory fixture patterns. +- Existing `beginFeedAttempt` / `attempt.finish` lifecycle inside provider loops. + +Risk and blast radius: + +- ASN providers are supporting databases for ASN comparison artifacts, entity details, IP context, homepage summaries, and integrity repair paths. +- Refactoring must not change when stale staged sources are accepted or how stale cached provider loads are recorded. +- Loaded database handles must still be closed if a later provider fails or context cancellation interrupts the run. +- Missing or malformed providers must keep current non-fatal versus fatal behavior: config, extraction, and missing database errors mark the provider and continue, while open failures return an error after cleanup. +- No downloader, scheduler queueing, public serving fallback, install behavior, UI behavior, or ASN attribution algorithm should change. + +Sensitive data handling plan: + +- This slice uses only local source code, synthetic ASN fixtures, temporary paths, and local posture/coverage metrics. +- No secrets, tokens, cookies, private endpoints, customer data, or personal data are needed. +- Durable artifacts will record only file paths, metrics, validation outcomes, and sanitized command evidence. + +Implementation plan: + +1. Introduce a focused ASN provider processing context that owns the source, provider directory paths, format spec, cache entry, reason, and time calculations. +2. Split per-provider processing into helpers for source metadata snapshotting, format validation, provider path construction, extraction/availability handling, database open, and loaded-stat recording. +3. Keep `processASNDatabases` as the orchestrator for source discovery, context cancellation checks, result map ownership, and close-on-error cleanup. +4. Add focused engine tests for no ASN sources, invalid ASN format, wrong-role format, missing database file, successful plain TSV provider load, and open failure cleanup/status behavior. +5. Preserve existing pipeline tests as broad integration coverage for ASN fan-out behavior. + +Validation plan: + +- Run `go test ./pkg/engine`. +- Run `go test -coverprofile=/tmp/update-ipsets-engine-slice25.cover -covermode=atomic ./pkg/engine` and inspect `go tool cover -func`. +- Run `go run ./tools/archposture -root . > /tmp/update-ipsets-archposture-slice25.json` and confirm `processASNDatabases` no longer appears as a production large-function target. +- Run `make lint`, `make staticcheck`, `make golangci-lint`, `CI=true make coverage`, and `make test-strict`. +- Run whitespace and durable-artifact forbidden-name scans over the changed files before commit. + +Artifact impact plan: + +- AGENTS.md: no update expected. +- Runtime project skills: update only if a repeatable ASN provider lifecycle lesson is found. +- Specs: no update expected because ASN provider lifecycle semantics are intended to stay unchanged. +- End-user/operator docs: no update expected. +- End-user/operator skills: no update expected. +- SOW lifecycle: this SOW remains in `.agents/sow/current/`; Slice 25 results will be recorded after validation. + +Open decisions: + +- No new user design decision is required because the slice is behavior-preserving quality work under the previously approved quality plan. ## Outcome -First through twenty-third implementation slices are complete, validated locally, and merged. The twenty-fourth implementation slice is complete and validated locally. The SOW remains open for the next focused coverage, complexity, or duplication slice. +First through twenty-fourth implementation slices are complete, validated locally, and merged. The twenty-fifth implementation slice is complete and validated locally. The SOW remains open for the next focused coverage, complexity, or duplication slice. ## Lessons Extracted diff --git a/pkg/engine/asn.go b/pkg/engine/asn.go index 0f89ebc..43acadd 100644 --- a/pkg/engine/asn.go +++ b/pkg/engine/asn.go @@ -6,10 +6,8 @@ import ( "os" "path/filepath" "sort" - "time" "github.com/firehol/update-ipsets/pkg/asnloc" - "github.com/firehol/update-ipsets/pkg/cache" "github.com/firehol/update-ipsets/pkg/config" "github.com/firehol/update-ipsets/pkg/iprange" ) @@ -53,106 +51,13 @@ func (e *Engine) processASNDatabases(ctx context.Context, opts RunOptions) (asnD datasets.closeAll(e.logger) return nil, err } - name := src.Name - entry := e.state.Entry(name) - attempt := e.beginFeedAttempt(entry, reason) - var loopErr error - func() { - defer attempt.finish() - - entry.ApplyProviderSourceConfig(cache.ProviderSourceConfigSnapshot{ - Name: name, - Category: src.Category, - DefaultCategory: "asn", - Info: src.Info, - Maintainer: src.Maintainer, - MaintainerURL: src.MaintainerURL, - Frequency: src.Frequency, - URL: src.URL, - Downloader: src.Downloader, - DownloaderOptions: src.DownloaderOptions, - }) - - spec, ok := lookupFormat(src.Format) - if !ok || spec.role != formatRoleASN { - e.logger.Error("ASN source has unknown or wrong-role format", "name", name, "format", src.Format) - entry.MarkProviderConfigError("unknown ASN format " + src.Format) - return - } - - providerDir := filepath.Join(asnDir, name) - if err := os.MkdirAll(providerDir, generatedDirMode); err != nil { - entry.MarkProviderFilesystemFailure(err.Error()) - loopErr = err - return - } - archivePath := filepath.Join(providerDir, "source") - processingArchivePath := preferStagedPath(archivePath) - dataPath := filepath.Join(providerDir, spec.dataFile) - archiveTime := time.Time{} - if archiveTime.IsZero() { - if info, err := os.Stat(processingArchivePath); err == nil { - archiveTime = info.ModTime().UTC() - } - } - if processingArchivePath != archivePath && spec.extract != nil { - entry.MarkProviderProcessing() - if err := spec.extract(processingArchivePath, dataPath); err != nil { - e.logger.Error("ASN staged extract failed", "name", name, "error", err) - entry.MarkProviderExtractFailed(err.Error()) - return - } - } else if !fileExists(dataPath) && spec.extract != nil { - entry.MarkProviderProcessing() - if err := spec.extract(processingArchivePath, dataPath); err != nil { - e.logger.Error("ASN extract failed", "name", name, "error", err) - entry.MarkProviderExtractFailed(err.Error()) - return - } - } - if !fileExists(dataPath) { - e.logger.Warn("ASN database not available, skipping source", "name", name, "path", dataPath) - entry.MarkProviderUnavailable("database file not found at " + dataPath) - return - } - entry.MarkProviderProcessing() - db, err := asnloc.Open(src.Format, dataPath) - if err != nil { - e.logger.Error("ASN open failed", "name", name, "format", src.Format, "path", dataPath, "error", err) - entry.MarkProviderOpenFailed(err.Error()) - loopErr = fmt.Errorf("asn open %s: %w", name, err) - return - } - datasets[name] = db - entries := entry.Entries - uniqueIPs := entry.UniqueIPs - if networks, ipv4Covered, statsErr := db.Stats(); statsErr != nil { - e.logger.Warn("ASN stats failed", "name", name, "error", statsErr) - } else { - entries = networks - uniqueIPs = ipv4Covered - } - processedAt := e.now().UTC() - now := e.now().UTC() - clockSkewSeconds := int64(0) - if archiveTime.After(now) { - clockSkewSeconds = int64(archiveTime.Sub(now).Seconds()) - } - stale := entry.RecordProviderLoaded(cache.ProviderLoadStats{ - SourceUnix: archiveTime.Unix(), - ProcessedUnix: processedAt.Unix(), - ClockSkewSeconds: clockSkewSeconds, - Entries: entries, - UniqueIPs: uniqueIPs, - }, src.Frequency, processingArchivePath != archivePath) - e.logger.Info("ASN source loaded", "name", name, "networks", entry.Entries, "ipv4_covered", entry.UniqueIPs) - if stale { - e.logger.Warn("ASN using stale data after download failure", "name", name, "failures", entry.DownloadFailures) - } - }() - if loopErr != nil { + db, err := e.processASNProvider(src, asnDir, reason) + if err != nil { datasets.closeAll(e.logger) - return nil, loopErr + return nil, err + } + if db != nil { + datasets[src.Name] = db } } return datasets, nil diff --git a/pkg/engine/asn_provider_processing.go b/pkg/engine/asn_provider_processing.go new file mode 100644 index 0000000..44348cc --- /dev/null +++ b/pkg/engine/asn_provider_processing.go @@ -0,0 +1,176 @@ +package engine + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/firehol/update-ipsets/pkg/asnloc" + "github.com/firehol/update-ipsets/pkg/cache" + "github.com/firehol/update-ipsets/pkg/config" + "github.com/firehol/update-ipsets/pkg/runreason" +) + +type asnProviderProcessor struct { + e *Engine + src *config.Source + entry *cache.Entry + reason runreason.Reason + asnDir string + spec formatSpec + archivePath string + processingArchivePath string + dataPath string + archiveTime time.Time +} + +func (e *Engine) processASNProvider(src *config.Source, asnDir string, reason runreason.Reason) (*asnloc.Database, error) { + processor := &asnProviderProcessor{ + e: e, + src: src, + entry: e.state.Entry(src.Name), + reason: reason, + asnDir: asnDir, + } + return processor.process() +} + +func (p *asnProviderProcessor) process() (*asnloc.Database, error) { + attempt := p.e.beginFeedAttempt(p.entry, p.reason) + defer attempt.finish() + + p.applySourceConfig() + if !p.validateFormat() { + return nil, nil + } + if err := p.preparePaths(); err != nil { + return nil, err + } + if !p.extractDatabaseIfNeeded() || !p.databaseAvailable() { + return nil, nil + } + db, err := p.openDatabase() + if err != nil { + return nil, err + } + p.recordLoaded(db) + return db, nil +} + +func (p *asnProviderProcessor) applySourceConfig() { + p.entry.ApplyProviderSourceConfig(cache.ProviderSourceConfigSnapshot{ + Name: p.src.Name, + Category: p.src.Category, + DefaultCategory: "asn", + Info: p.src.Info, + Maintainer: p.src.Maintainer, + MaintainerURL: p.src.MaintainerURL, + Frequency: p.src.Frequency, + URL: p.src.URL, + Downloader: p.src.Downloader, + DownloaderOptions: p.src.DownloaderOptions, + }) +} + +func (p *asnProviderProcessor) validateFormat() bool { + spec, ok := lookupFormat(p.src.Format) + if !ok || spec.role != formatRoleASN { + p.e.logger.Error("ASN source has unknown or wrong-role format", "name", p.src.Name, "format", p.src.Format) + p.entry.MarkProviderConfigError("unknown ASN format " + p.src.Format) + return false + } + p.spec = spec + return true +} + +func (p *asnProviderProcessor) preparePaths() error { + providerDir := filepath.Join(p.asnDir, p.src.Name) + if err := os.MkdirAll(providerDir, generatedDirMode); err != nil { + p.entry.MarkProviderFilesystemFailure(err.Error()) + return err + } + p.archivePath = filepath.Join(providerDir, "source") + p.processingArchivePath = preferStagedPath(p.archivePath) + p.dataPath = filepath.Join(providerDir, p.spec.dataFile) + if info, err := os.Stat(p.processingArchivePath); err == nil { + p.archiveTime = info.ModTime().UTC() + } + return nil +} + +func (p *asnProviderProcessor) extractDatabaseIfNeeded() bool { + if p.spec.extract == nil { + return true + } + switch { + case p.processingArchivePath != p.archivePath: + return p.extractDatabase("ASN staged extract failed") + case !fileExists(p.dataPath): + return p.extractDatabase("ASN extract failed") + default: + return true + } +} + +func (p *asnProviderProcessor) extractDatabase(logMessage string) bool { + p.entry.MarkProviderProcessing() + if err := p.spec.extract(p.processingArchivePath, p.dataPath); err != nil { + p.e.logger.Error(logMessage, "name", p.src.Name, "error", err) + p.entry.MarkProviderExtractFailed(err.Error()) + return false + } + return true +} + +func (p *asnProviderProcessor) databaseAvailable() bool { + if fileExists(p.dataPath) { + return true + } + p.e.logger.Warn("ASN database not available, skipping source", "name", p.src.Name, "path", p.dataPath) + p.entry.MarkProviderUnavailable("database file not found at " + p.dataPath) + return false +} + +func (p *asnProviderProcessor) openDatabase() (*asnloc.Database, error) { + p.entry.MarkProviderProcessing() + db, err := asnloc.Open(p.src.Format, p.dataPath) + if err != nil { + p.e.logger.Error("ASN open failed", "name", p.src.Name, "format", p.src.Format, "path", p.dataPath, "error", err) + p.entry.MarkProviderOpenFailed(err.Error()) + return nil, fmt.Errorf("asn open %s: %w", p.src.Name, err) + } + return db, nil +} + +func (p *asnProviderProcessor) recordLoaded(db *asnloc.Database) { + entries, uniqueIPs := p.providerStats(db) + processedAt := p.e.now().UTC() + now := p.e.now().UTC() + clockSkewSeconds := int64(0) + if p.archiveTime.After(now) { + clockSkewSeconds = int64(p.archiveTime.Sub(now).Seconds()) + } + stale := p.entry.RecordProviderLoaded(cache.ProviderLoadStats{ + SourceUnix: p.archiveTime.Unix(), + ProcessedUnix: processedAt.Unix(), + ClockSkewSeconds: clockSkewSeconds, + Entries: entries, + UniqueIPs: uniqueIPs, + }, p.src.Frequency, p.processingArchivePath != p.archivePath) + p.e.logger.Info("ASN source loaded", "name", p.src.Name, "networks", p.entry.Entries, "ipv4_covered", p.entry.UniqueIPs) + if stale { + p.e.logger.Warn("ASN using stale data after download failure", "name", p.src.Name, "failures", p.entry.DownloadFailures) + } +} + +func (p *asnProviderProcessor) providerStats(db *asnloc.Database) (int, uint64) { + entries := p.entry.Entries + uniqueIPs := p.entry.UniqueIPs + networks, ipv4Covered, err := db.Stats() + if err != nil { + p.e.logger.Warn("ASN stats failed", "name", p.src.Name, "error", err) + return entries, uniqueIPs + } + return networks, ipv4Covered +} diff --git a/pkg/engine/asn_provider_processing_test.go b/pkg/engine/asn_provider_processing_test.go new file mode 100644 index 0000000..ec479e6 --- /dev/null +++ b/pkg/engine/asn_provider_processing_test.go @@ -0,0 +1,241 @@ +package engine + +import ( + "compress/gzip" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/firehol/update-ipsets/pkg/config" +) + +const asnProcessingSampleTSV = `1.0.0.0 1.0.0.255 13335 US CLOUDFLARENET +8.8.8.0 8.8.8.255 15169 US GOOGLE +` + +func TestProcessASNDatabasesNoSources(t *testing.T) { + eng := newEngineFixture(t) + + datasets, err := eng.processASNDatabases(t.Context(), RunOptions{}) + if err != nil { + t.Fatalf("processASNDatabases() error = %v", err) + } + if datasets != nil { + t.Fatalf("datasets = %#v, want nil without ASN sources", datasets) + } +} + +func TestProcessASNDatabasesRejectsInvalidFormats(t *testing.T) { + tests := []struct { + name string + format string + }{ + {name: "unknown format", format: "not_registered"}, + {name: "wrong role format", format: "dbip_country_csv"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + src := asnProcessingSource("provider", tt.format) + eng := newASNProcessingEngine(t, src) + + datasets, err := eng.processASNDatabases(t.Context(), RunOptions{}) + if err != nil { + t.Fatalf("processASNDatabases() error = %v", err) + } + if len(datasets) != 0 { + t.Fatalf("datasets len = %d, want 0", len(datasets)) + } + entry := eng.state.Entry(src.Name) + if entry.LastStatus != "config_error" { + t.Fatalf("last status = %q, want config_error", entry.LastStatus) + } + if entry.LastError != "unknown ASN format "+tt.format { + t.Fatalf("last error = %q", entry.LastError) + } + if entry.Category != "asn" || entry.Info != "ASN provider" || entry.MaintainerURL != "https://example.test/provider" { + t.Fatalf("provider source metadata was not applied: %+v", entry) + } + }) + } +} + +func TestProcessASNDatabasesMarksUnavailableWhenDatabaseMissing(t *testing.T) { + format := "test_asn_no_extract" + withASNProcessingFormat(t, format, formatSpec{role: formatRoleASN, dataFile: "database.tsv"}) + src := asnProcessingSource("provider", format) + eng := newASNProcessingEngine(t, src) + + datasets, err := eng.processASNDatabases(t.Context(), RunOptions{}) + if err != nil { + t.Fatalf("processASNDatabases() error = %v", err) + } + if len(datasets) != 0 { + t.Fatalf("datasets len = %d, want 0", len(datasets)) + } + entry := eng.state.Entry(src.Name) + if entry.LastStatus != "unavailable" { + t.Fatalf("last status = %q, want unavailable", entry.LastStatus) + } + if !strings.Contains(entry.LastError, "database file not found") { + t.Fatalf("last error = %q, want database file not found", entry.LastError) + } +} + +func TestProcessASNDatabasesLoadsExistingDatabase(t *testing.T) { + src := asnProcessingSource("provider", "iptoasn_combined_tsv") + eng := newASNProcessingEngine(t, src) + writeASNProcessingDatabase(t, eng, src.Name, "database.tsv", asnProcessingSampleTSV) + + datasets, err := eng.processASNDatabases(t.Context(), RunOptions{}) + if err != nil { + t.Fatalf("processASNDatabases() error = %v", err) + } + defer datasets.closeAll(eng.logger) + if len(datasets) != 1 || datasets[src.Name] == nil { + t.Fatalf("datasets = %#v, want one loaded provider", datasets) + } + entry := eng.state.Entry(src.Name) + if entry.LastStatus != "updated" || entry.LastError != "" { + t.Fatalf("provider status = %q error %q, want updated without error", entry.LastStatus, entry.LastError) + } + if entry.Entries != 2 || entry.UniqueIPs != 512 { + t.Fatalf("provider stats = entries %d unique %d, want 2 and 512", entry.Entries, entry.UniqueIPs) + } + if entry.FrequencyMinutes != src.Frequency || entry.PublicURL != src.URL { + t.Fatalf("provider metadata = %+v, want configured frequency and URL", entry) + } +} + +func TestProcessASNDatabasesUsesStagedArchiveAsFreshLoad(t *testing.T) { + src := asnProcessingSource("provider", "iptoasn_combined_tsv") + eng := newASNProcessingEngine(t, src) + entry := eng.state.Entry(src.Name) + entry.DownloadFailures = 3 + + providerDir := filepath.Join(eng.runtime.LibDir, "asn", src.Name) + if err := os.MkdirAll(providerDir, 0o700); err != nil { + t.Fatal(err) + } + stagedArchive := stagedPath(filepath.Join(providerDir, "source")) + writeGzipASNProcessingFixture(t, stagedArchive, asnProcessingSampleTSV) + archiveTime := time.Unix(2500, 0).UTC() + if err := os.Chtimes(stagedArchive, archiveTime, archiveTime); err != nil { + t.Fatal(err) + } + + datasets, err := eng.processASNDatabases(t.Context(), RunOptions{}) + if err != nil { + t.Fatalf("processASNDatabases() error = %v", err) + } + defer datasets.closeAll(eng.logger) + if len(datasets) != 1 { + t.Fatalf("datasets len = %d, want 1", len(datasets)) + } + if entry.LastStatus != "updated" || entry.LastError != "" { + t.Fatalf("provider status = %q error %q, want staged load to clear stale failure", entry.LastStatus, entry.LastError) + } + if entry.SourceDate != archiveTime.Unix() { + t.Fatalf("source date = %d, want %d", entry.SourceDate, archiveTime.Unix()) + } + if !fileExists(filepath.Join(providerDir, "database.tsv")) { + t.Fatal("staged archive was not extracted to database.tsv") + } +} + +func TestProcessASNDatabasesReturnsOpenFailure(t *testing.T) { + good := asnProcessingSource("aa_good", "iptoasn_combined_tsv") + bad := asnProcessingSource("zz_bad", "iptoasn_combined_tsv") + eng := newASNProcessingEngine(t, good, bad) + writeASNProcessingDatabase(t, eng, good.Name, "database.tsv", asnProcessingSampleTSV) + writeASNProcessingDatabase(t, eng, bad.Name, "database.tsv", "1.2.3.0\t1.2.3.255\t13335\n") + + datasets, err := eng.processASNDatabases(t.Context(), RunOptions{}) + if err == nil { + t.Fatal("processASNDatabases() error = nil, want open failure") + } + if !strings.Contains(err.Error(), "asn open zz_bad") { + t.Fatalf("error = %v, want zz_bad open context", err) + } + if datasets != nil { + t.Fatalf("datasets = %#v, want nil on fatal open error", datasets) + } + if got := eng.state.Entry(good.Name).LastStatus; got != "updated" { + t.Fatalf("good provider status = %q, want updated before later failure", got) + } + if got := eng.state.Entry(bad.Name).LastStatus; got != "open_failed" { + t.Fatalf("bad provider status = %q, want open_failed", got) + } +} + +func newASNProcessingEngine(t *testing.T, sources ...*config.Source) *Engine { + t.Helper() + cfg := config.New() + cfg.SourceOrder = make([]string, 0, len(sources)) + for _, src := range sources { + cfg.Sources[src.Name] = src + cfg.SourceOrder = append(cfg.SourceOrder, src.Name) + } + return newEngineFixture(t, withConfig(cfg), withNow(func() time.Time { + return time.Unix(2000, 0).UTC() + })) +} + +func asnProcessingSource(name, format string) *config.Source { + return &config.Source{ + Name: name, + URL: "https://example.test/" + name, + Use: []string{config.UseASN}, + Format: format, + Frequency: 1440, + Info: "ASN provider", + Maintainer: "Example maintainer", + MaintainerURL: "https://example.test/provider", + } +} + +func writeASNProcessingDatabase(t *testing.T, eng *Engine, provider, file, body string) { + t.Helper() + providerDir := filepath.Join(eng.runtime.LibDir, "asn", provider) + if err := os.MkdirAll(providerDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(providerDir, file), []byte(body), 0o600); err != nil { + t.Fatal(err) + } +} + +func writeGzipASNProcessingFixture(t *testing.T, path, body string) { + t.Helper() + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + if err != nil { + t.Fatal(err) + } + gz := gzip.NewWriter(f) + if _, err := gz.Write([]byte(body)); err != nil { + _ = gz.Close() + _ = f.Close() + t.Fatal(err) + } + if err := gz.Close(); err != nil { + _ = f.Close() + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } +} + +func withASNProcessingFormat(t *testing.T, name string, spec formatSpec) { + t.Helper() + old, existed := formatRegistry[name] + formatRegistry[name] = spec + t.Cleanup(func() { + if existed { + formatRegistry[name] = old + return + } + delete(formatRegistry, name) + }) +}