diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cc83c4..c634ce9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -187,6 +187,56 @@ A new profile field, `layout_unit_rule`, `line-units-v1` (the office profiles na twelve, sixteen and twenty-four and above a share of the unit's width; in the engine a gap of one line height between pieces scored above a half, seven tenths, one and a half and two. +### A line read where the reading order reads it, and its words where the page set them apart (`line-units-v5`, decision #67) + +`layout_unit_rule` moves `line-units-v4` -> `line-units-v5`, so `profile_sha256` moves. + +- **A line is read where the reading order reads it.** On a page the reading order did not divide, + a line was every run on its baseline across the page, read where its first run was: a paragraph + whose baselines a byline or a caption in the next column shares was read where that column was, + and fell into several units (a ten-line newspaper paragraph read as five). Each line is now cut + into its pieces before any unit is read, and a line as the units read it is the pieces of one + line that follow each other in reading order. +- **A word gap the page set by moving its pen reads as a space** where the rule reads a line's + text: wherever two runs' boxes stand a tenth of the line's height apart or more. A bullet set + apart by its position, with no space drawn after it, now opens a list item, and the hard-break + cut reads a line's first word as a word rather than as the whole line. +- **Measured** on ParseBench: visual grounding 0.5435 -> 0.5575, 119 pages up and 8 down; content + one page up by 0.0003; tables, charts and formatting unchanged on every page; overall 46.42 -> + 46.70. opendataloader-bench, the fabrication check and the gate documents' tables unchanged. +- **The trade, taken by the owner**: the line spanning two columns gave each unit a far right edge, + and that edge cut some units by accident — a contents list's entries (`Marico_Annual_Report_FY24` + p4, twice in the set), a paragraph's last line, a label's — which now join; a bullet list the + ground truth keeps as one block (`entire-full-report-basf-ar24` p17) is a unit per bullet; and a + chart's labels join the text beside them, so its picture holds no text. + +### A line chart's printed values read as its table (`point-labels-v1`), and a panel's title names its numbers (`bar-labels-v3`), decision #66 + +A new profile field, `table_detection.lines` = `point-labels-v1` (office profiles: +`not-run-for-this-format`), and `table_detection.charts` `bar-labels-v2` -> `-v3`, so +`profile_sha256` moves. `docs/32-CHART-LABELS-SCOPE.md` §8 and §9 are the rules. + +- **A line chart that prints a number at each point is a table**: a stroked line rising left to + right, or a run of three or more small marks of one colour, is a series; each number goes to the + one point it is set over, under or beside, unless another series' point stands nearly as near; a + year is never a value; a series keeps its numbers where six points in ten carry one and a higher + point never carries a smaller number; its categories are the first row of labels under the plot. + Series sharing that row are one table, each named by its legend swatch (a small mark or a short + stroke of its colour) or, alone, by the title over it. A value is never read off a point's + height. +- **The interpreter carries the stroke colour** (`G`, `RG`, `K`, `SC`, `SCN`, reset by `CS`, saved + by `q`) and records each stroked subpath's points and each small filled mark with its colour; + only the line rule reads them. +- **`bar-labels-v3`**: the title over a chart names the one series that prints numbers, whatever + else is drawn beside it — a progress bar's track no longer leaves its numbers unnamed. +- **Measured** on ParseBench: charts 0.1184 -> 0.1248, 7 pages up and none down (`She-figures` p88 + and `E-Government_Survey` p78 0 -> 1.0); tables, content, formatting and visual grounding + unchanged on every page; overall 46.29 -> 46.42. No line table on a table or text page; the gate + documents' tables unchanged. +- **The trade, taken by the owner**: opendataloader-bench `01030000000078`'s line, read right + against its drawing, reorders that document's labels — its NID 0.8786 -> 0.8099 and TEDS 0.8667 -> + 0.7086 (bench NID 0.8841 -> 0.8838, TEDS 0.5339 -> 0.5301), as decision #42's bar tables did. + ### A heading's section number and its title are one line, and a label's lines one block (decision #65) Folded into `markdown-blocks-v12` and `html-blocks-v12`, which no release carries yet, so no rule id diff --git a/crates/ethos-parser-cli/tests/public_api.rs b/crates/ethos-parser-cli/tests/public_api.rs index 5672a68..54ab3ac 100644 --- a/crates/ethos-parser-cli/tests/public_api.rs +++ b/crates/ethos-parser-cli/tests/public_api.rs @@ -326,6 +326,7 @@ const CORE: &[&str] = &[ "LAYOUT_UNIT_RULE_V2", "LAYOUT_UNIT_RULE_V3", "LAYOUT_UNIT_RULE_V4", + "LAYOUT_UNIT_RULE_V5", // Decision #41. The page-furniture rule id, and the edge a furniture run stands apart at. "FURNITURE_RULE_V1", "Furniture", @@ -363,6 +364,9 @@ const CORE: &[&str] = &[ "MARKDOWN_SCHEMA_VERSION", "TABLE_DETECTION_CHARTS_V1", "TABLE_DETECTION_CHARTS_V2", + // Decision #66. + "TABLE_DETECTION_CHARTS_V3", + "TABLE_DETECTION_LINES_V1", "TABLE_DETECTION_STROKE_V1", "TABLE_DETECTION_TAGGED_V1", "TABLE_DETECTION_TRACKS_V3", diff --git a/crates/ethos-parser-core/src/lib.rs b/crates/ethos-parser-core/src/lib.rs index b06f60f..8434290 100644 --- a/crates/ethos-parser-core/src/lib.rs +++ b/crates/ethos-parser-core/src/lib.rs @@ -125,23 +125,23 @@ pub use profile::{ EPUB_READING_ORDER_RULE_V1, EPUB_TEXT_CODE_RULE_V2, FIGURE_RULE_V1, FORM_ANNOTATION_RULE_V2, FURNITURE_RULE_V1, HEADING_INFERENCE_RULE_V4, HEADING_INFERENCE_RULE_V5, HEADING_INFERENCE_RULE_V6, HEADING_INFERENCE_RULE_V7, LAYOUT_UNIT_RULE_V1, LAYOUT_UNIT_RULE_V2, - LAYOUT_UNIT_RULE_V3, LAYOUT_UNIT_RULE_V4, NOT_RUN, OBSERVATION_RULE_V3, OBSERVATION_RULE_V4, - ODP_READING_ORDER_RULE_V1, ODP_TEXT_CODE_RULE_V2, ODS_READING_ORDER_RULE_V1, - ODS_TEXT_CODE_RULE_V2, ODT_READING_ORDER_RULE_V1, ODT_TEXT_CODE_RULE_V2, - PPTX_READING_ORDER_RULE_V1, PPTX_TEXT_CODE_RULE_V3, READING_ORDER_RULE_V0, - READING_ORDER_RULE_V1, READING_ORDER_RULE_V2, READING_ORDER_RULE_V3, READING_ORDER_RULE_V4, - RIGHT_TO_LEFT_RULE_V1, RTF_READING_ORDER_RULE_V1, RTF_TEXT_CODE_RULE_V2, STRUCT_TREE_RULE_V1, - STRUCT_TREE_RULE_V2, TABLE_DETECTION_CHARTS_V1, TABLE_DETECTION_CHARTS_V2, - TABLE_DETECTION_STROKE_V1, TABLE_DETECTION_TAGGED_V1, TABLE_DETECTION_TRACKS_V10, - TABLE_DETECTION_TRACKS_V11, TABLE_DETECTION_TRACKS_V12, TABLE_DETECTION_TRACKS_V13, - TABLE_DETECTION_TRACKS_V14, TABLE_DETECTION_TRACKS_V3, TABLE_DETECTION_TRACKS_V4, - TABLE_DETECTION_TRACKS_V5, TABLE_DETECTION_TRACKS_V6, TABLE_DETECTION_TRACKS_V7, - TABLE_DETECTION_TRACKS_V8, TABLE_DETECTION_TRACKS_V9, TABLE_DETECTION_UNRULED_V1, - TABLE_DETECTION_V1, TABLE_DETECTION_V10, TABLE_DETECTION_V11, TABLE_DETECTION_V12, - TABLE_DETECTION_V2, TABLE_DETECTION_V3, TABLE_DETECTION_V4, TABLE_DETECTION_V5, - TABLE_DETECTION_V6, TABLE_DETECTION_V7, TABLE_DETECTION_V8, TABLE_DETECTION_V9, - TEXT_CODE_RULE_V2, TEXT_CODE_RULE_V3, TEXT_CODE_RULE_V4, XLSX_READING_ORDER_RULE_V1, - XLSX_TEXT_CODE_RULE_V3, + LAYOUT_UNIT_RULE_V3, LAYOUT_UNIT_RULE_V4, LAYOUT_UNIT_RULE_V5, NOT_RUN, OBSERVATION_RULE_V3, + OBSERVATION_RULE_V4, ODP_READING_ORDER_RULE_V1, ODP_TEXT_CODE_RULE_V2, + ODS_READING_ORDER_RULE_V1, ODS_TEXT_CODE_RULE_V2, ODT_READING_ORDER_RULE_V1, + ODT_TEXT_CODE_RULE_V2, PPTX_READING_ORDER_RULE_V1, PPTX_TEXT_CODE_RULE_V3, + READING_ORDER_RULE_V0, READING_ORDER_RULE_V1, READING_ORDER_RULE_V2, READING_ORDER_RULE_V3, + READING_ORDER_RULE_V4, RIGHT_TO_LEFT_RULE_V1, RTF_READING_ORDER_RULE_V1, RTF_TEXT_CODE_RULE_V2, + STRUCT_TREE_RULE_V1, STRUCT_TREE_RULE_V2, TABLE_DETECTION_CHARTS_V1, TABLE_DETECTION_CHARTS_V2, + TABLE_DETECTION_CHARTS_V3, TABLE_DETECTION_LINES_V1, TABLE_DETECTION_STROKE_V1, + TABLE_DETECTION_TAGGED_V1, TABLE_DETECTION_TRACKS_V10, TABLE_DETECTION_TRACKS_V11, + TABLE_DETECTION_TRACKS_V12, TABLE_DETECTION_TRACKS_V13, TABLE_DETECTION_TRACKS_V14, + TABLE_DETECTION_TRACKS_V3, TABLE_DETECTION_TRACKS_V4, TABLE_DETECTION_TRACKS_V5, + TABLE_DETECTION_TRACKS_V6, TABLE_DETECTION_TRACKS_V7, TABLE_DETECTION_TRACKS_V8, + TABLE_DETECTION_TRACKS_V9, TABLE_DETECTION_UNRULED_V1, TABLE_DETECTION_V1, TABLE_DETECTION_V10, + TABLE_DETECTION_V11, TABLE_DETECTION_V12, TABLE_DETECTION_V2, TABLE_DETECTION_V3, + TABLE_DETECTION_V4, TABLE_DETECTION_V5, TABLE_DETECTION_V6, TABLE_DETECTION_V7, + TABLE_DETECTION_V8, TABLE_DETECTION_V9, TEXT_CODE_RULE_V2, TEXT_CODE_RULE_V3, + TEXT_CODE_RULE_V4, XLSX_READING_ORDER_RULE_V1, XLSX_TEXT_CODE_RULE_V3, }; pub use tables::{ CellSlot, CheckStatus, GeometricFault, LocatorCheck, SlotCover, SlotFault, TableCellPosition, diff --git a/crates/ethos-parser-core/src/profile.rs b/crates/ethos-parser-core/src/profile.rs index c3a1424..f7dd4be 100644 --- a/crates/ethos-parser-core/src/profile.rs +++ b/crates/ethos-parser-core/src/profile.rs @@ -907,6 +907,30 @@ pub const TABLE_DETECTION_CHARTS_V1: &str = "bar-labels-v1"; /// read. A value is still never read off a bar's length. pub const TABLE_DETECTION_CHARTS_V2: &str = "bar-labels-v2"; +/// [`TABLE_DETECTION_CHARTS_V2`] **with the title over a chart naming the one series that prints +/// numbers** (decision #66, `docs/32-CHART-LABELS-SCOPE.md` §8). +/// +/// `-v2` read a chart's title as its series' name only where the chart was drawn in one colour. A +/// chart of bars drawn over a full-length track — a progress bar, one colour printing its numbers +/// and one printing none — kept two series, and its one column of numbers went unnamed, though the +/// panel's title above it names it. `-v3` reads the title for the one series that carries numbers, +/// whatever else is drawn beside it. +pub const TABLE_DETECTION_CHARTS_V3: &str = "bar-labels-v3"; + +/// The rule that reads **the numbers a line chart prints at its points back as its table** +/// (decision #66, `docs/32-CHART-LABELS-SCOPE.md` §9). +/// +/// A series is a line a page stroked whose every point stands right of the one before, or a run +/// of three or more small marks of one colour and size so set; each number on the page goes to the +/// one point it stands over, under or beside, unless another series' point stands nearly as near. +/// A series keeps its numbers where six points in ten carry one and a point higher on the page +/// never carries a smaller number; its categories are the first row of labels below the plot with +/// one label under each point; series sharing one row are one table, each named by the label +/// beside a legend swatch of its colour, or a lone series by the title over it. A value is never +/// read off a point's height. Runs after [`TABLE_DETECTION_CHARTS_V3`], on runs no other rule's +/// table holds, only on a document that declares no author structure. +pub const TABLE_DETECTION_LINES_V1: &str = "point-labels-v1"; + /// The structure-tree rule v1-S3 ships: read `/StructTreeRoot`, bind by `(page, mcid)`. /// /// On the profile because it changes output. Which structure types are recognised, how `/RoleMap` @@ -1165,6 +1189,24 @@ pub const LAYOUT_UNIT_RULE_V3: &str = "line-units-v3"; /// figure, opening where the figure opens, stays a line of its own. Units stay where, never what. pub const LAYOUT_UNIT_RULE_V4: &str = "line-units-v4"; +/// [`LAYOUT_UNIT_RULE_V4`] **with each line read where the reading order reads it, and its words +/// where the page set them apart** (decision #67). +/// +/// **What `-v4` missed.** Told "Grounding: text, round 2", the `Text` elements ParseBench's visual +/// grounding could not place were read for why, and the largest share, 811 of them, lay split +/// across several of our units. On a page the reading order did not divide, a line was every run +/// on its baseline across the page, read where its first run was: a paragraph whose baselines a +/// byline or a caption in the next column shares was read where that column was, and its lines +/// fell into several units. And a page that sets its words by moving its pen draws no space +/// between them, so a bullet set apart that way opened no list item, and the hard-break cut read a +/// line's whole width as its first word. +/// +/// **The change.** Each line is cut into its pieces before any unit is read, and a line as the +/// units read it is the pieces of one line that follow each other in reading order. The text the +/// list-marker clause and the first word are read from has a space wherever two runs' boxes stand +/// a tenth of the line's height apart or more. Units stay where, never what. +pub const LAYOUT_UNIT_RULE_V5: &str = "line-units-v5"; + /// The page-furniture rule decision #41 ships: **what a page sets apart at its top and bottom /// edges**. /// @@ -1438,6 +1480,13 @@ pub struct TableDetection { /// See [`TABLE_DETECTION_CHARTS_V1`]. A sixth field for the reason there were five: its /// evidence — filled bars, their colours, and the labels set at them — is its own. pub charts: String, + /// Version id of the rule that reads the numbers a line chart prints at its points as its table + /// (decision #66). + /// + /// See [`TABLE_DETECTION_LINES_V1`]. A seventh field for the reason there were six: its + /// evidence — stroked lines and small marks, their colours, and the labels set at their points — + /// is its own. + pub lines: String, } impl Default for TableDetection { @@ -1449,7 +1498,8 @@ impl Default for TableDetection { stroke_ruled: TABLE_DETECTION_STROKE_V1.to_string(), tagged: TABLE_DETECTION_TAGGED_V1.to_string(), tracks: TABLE_DETECTION_TRACKS_V14.to_string(), - charts: TABLE_DETECTION_CHARTS_V2.to_string(), + charts: TABLE_DETECTION_CHARTS_V3.to_string(), + lines: TABLE_DETECTION_LINES_V1.to_string(), } } } @@ -2165,7 +2215,7 @@ pub struct Profile { pub heading_inference_rule: String, /// Version id of the layout-unit rule in force (decision #38). /// - /// See [`LAYOUT_UNIT_RULE_V4`]. Its own field for `heading_inference_rule`'s reason: it runs + /// See [`LAYOUT_UNIT_RULE_V5`]. Its own field for `heading_inference_rule`'s reason: it runs /// only where the document declares no structure, so folding it into a rule that runs on every /// document would make tagged artifacts non-comparable across a change to a rule that never /// ran on them. Refused, not defaulted, when absent. @@ -2276,7 +2326,7 @@ impl Default for Profile { outline_rule: OUTLINE_RULE_V3.to_string(), text_box_rule: TEXT_BOX_RULE_V1.to_string(), heading_inference_rule: HEADING_INFERENCE_RULE_V7.to_string(), - layout_unit_rule: LAYOUT_UNIT_RULE_V4.to_string(), + layout_unit_rule: LAYOUT_UNIT_RULE_V5.to_string(), furniture_rule: FURNITURE_RULE_V1.to_string(), figure_rule: FIGURE_RULE_V1.to_string(), right_to_left_rule: RIGHT_TO_LEFT_RULE_V1.to_string(), @@ -2366,6 +2416,7 @@ impl Profile { tagged: NOT_RUN.into(), tracks: NOT_RUN.into(), charts: NOT_RUN.into(), + lines: NOT_RUN.into(), }, reading_order_rule: DOCX_READING_ORDER_RULE_V2.to_string(), struct_tree_rule: NOT_RUN.into(), @@ -2444,6 +2495,7 @@ impl Profile { tagged: NOT_RUN.into(), tracks: NOT_RUN.into(), charts: NOT_RUN.into(), + lines: NOT_RUN.into(), }, reading_order_rule: XLSX_READING_ORDER_RULE_V1.to_string(), struct_tree_rule: NOT_RUN.into(), @@ -2516,6 +2568,7 @@ impl Profile { tagged: NOT_RUN.into(), tracks: NOT_RUN.into(), charts: NOT_RUN.into(), + lines: NOT_RUN.into(), }, reading_order_rule: PPTX_READING_ORDER_RULE_V1.to_string(), struct_tree_rule: NOT_RUN.into(), @@ -2592,6 +2645,7 @@ impl Profile { tagged: NOT_RUN.into(), tracks: NOT_RUN.into(), charts: NOT_RUN.into(), + lines: NOT_RUN.into(), }, reading_order_rule: ODT_READING_ORDER_RULE_V1.to_string(), struct_tree_rule: NOT_RUN.into(), @@ -2655,6 +2709,7 @@ impl Profile { tagged: NOT_RUN.into(), tracks: NOT_RUN.into(), charts: NOT_RUN.into(), + lines: NOT_RUN.into(), }, reading_order_rule: ODS_READING_ORDER_RULE_V1.to_string(), struct_tree_rule: NOT_RUN.into(), @@ -2724,6 +2779,7 @@ impl Profile { tagged: NOT_RUN.into(), tracks: NOT_RUN.into(), charts: NOT_RUN.into(), + lines: NOT_RUN.into(), }, reading_order_rule: ODP_READING_ORDER_RULE_V1.to_string(), struct_tree_rule: NOT_RUN.into(), @@ -2792,6 +2848,7 @@ impl Profile { tagged: NOT_RUN.into(), tracks: NOT_RUN.into(), charts: NOT_RUN.into(), + lines: NOT_RUN.into(), }, reading_order_rule: RTF_READING_ORDER_RULE_V1.to_string(), struct_tree_rule: NOT_RUN.into(), @@ -2863,6 +2920,7 @@ impl Profile { tagged: NOT_RUN.into(), tracks: NOT_RUN.into(), charts: NOT_RUN.into(), + lines: NOT_RUN.into(), }, reading_order_rule: EPUB_READING_ORDER_RULE_V1.to_string(), struct_tree_rule: NOT_RUN.into(), @@ -3052,6 +3110,7 @@ mod tests { tagged, tracks, charts, + lines, }) ); assert_eq!( @@ -3153,6 +3212,7 @@ mod tests { tagged: _, tracks: _, charts: _, + lines: _, }, struct_tree_rule: _, outline_rule: _, @@ -3220,6 +3280,10 @@ mod tests { "table_detection.charts", Box::new(|p: &mut Profile| p.table_detection.charts = "other-charts-v9".into()), ), + ( + "table_detection.lines", + Box::new(|p: &mut Profile| p.table_detection.lines = "other-lines-v9".into()), + ), ( // v1-S4. Which form and annotation nodes exist at all. "form_annotation_rule", @@ -3437,12 +3501,13 @@ mod tests { // `furniture_rule` the same way, and forty-two since decision #42, which added // `table_detection.charts` the same way. Forty-four since decision #46, which added // `figure_rule` and `capabilities.figures` the same way, and forty-five since decision - // #54, which added `right_to_left_rule` the same way. + // #54, which added `right_to_left_rule` the same way. Forty-six since decision #66, which + // added `table_detection.lines` the same way. assert_eq!( mutations.len(), - 45, - "{} single-field mutation(s); forty-five is the number at decision #54, which added \ - `right_to_left_rule` with its mutation", + 46, + "{} single-field mutation(s); forty-six is the number at decision #66, which added \ + `table_detection.lines` with its mutation", mutations.len() ); @@ -3497,7 +3562,7 @@ mod tests { let bytes = Profile::default().canonical_bytes().unwrap(); assert_eq!( String::from_utf8(bytes).unwrap(), - r#"{"backend":{"components":{"flate2":"1.1.9","read-fonts":"0.36.0","skrifa":"0.39.0","unicode-bidi":"0.3.18"},"name":"lopdf","version":"0.44.0"},"capabilities":{"annotations":true,"char_offsets":true,"figures":true,"form_fields":true,"html":true,"images":true,"markdown":true,"measured_ink_boxes":true,"multi_column_reading_order":true,"outlines":true,"page_screenshots":false,"spans":true,"structural_locators":true,"tables":true},"classify_sample_pages":8,"cmap_data_version":"annex-d-encodings-4","coordinate_system":{"origin":"top-left","unit":"centipoint"},"figure_rule":"figure-regions-v1","font_metrics_data_version":"core14-afm-2","form_annotation_rule":"form-annotations-v2","furniture_rule":"margin-bands-v1","heading_inference_rule":"type-size-v7","html_rule":"html-blocks-v12","layout_unit_rule":"line-units-v4","locate_rule":"locate-scalar-exact-v1","markdown_rule":"markdown-blocks-v12","observation_rule":"page-observations-v4","outline_rule":"outlines-v3","page_budget":{"mode":"unlimited"},"parser_version":"0.64.0-dev.1","quantum_per_point":100,"raster_dpi":{"mode":"not_emitted"},"reading_order_rule":"gutter-columns-v4","right_to_left_rule":"right-to-left-lines-v1","struct_tree_rule":"struct-tree-v2","table_detection":{"charts":"bar-labels-v2","ruled":"ruled-rects-v12","stroke_ruled":"stroke-ruled-v1","tagged":"tagged-tables-v1","tracks":"whitespace-tracks-v14","unruled":"unruled-align-v1"},"text_box_rule":"advance-over-font-envelope-v1","text_code_rule":"declared-font-codes-v4","unicode_data_version":"std-unicode-16.0.0","verifier":{"mode":"not_pinned"},"xref_repair":{"mode":"pad-19-to-20-v1"}}"#, + r#"{"backend":{"components":{"flate2":"1.1.9","read-fonts":"0.36.0","skrifa":"0.39.0","unicode-bidi":"0.3.18"},"name":"lopdf","version":"0.44.0"},"capabilities":{"annotations":true,"char_offsets":true,"figures":true,"form_fields":true,"html":true,"images":true,"markdown":true,"measured_ink_boxes":true,"multi_column_reading_order":true,"outlines":true,"page_screenshots":false,"spans":true,"structural_locators":true,"tables":true},"classify_sample_pages":8,"cmap_data_version":"annex-d-encodings-4","coordinate_system":{"origin":"top-left","unit":"centipoint"},"figure_rule":"figure-regions-v1","font_metrics_data_version":"core14-afm-2","form_annotation_rule":"form-annotations-v2","furniture_rule":"margin-bands-v1","heading_inference_rule":"type-size-v7","html_rule":"html-blocks-v12","layout_unit_rule":"line-units-v5","locate_rule":"locate-scalar-exact-v1","markdown_rule":"markdown-blocks-v12","observation_rule":"page-observations-v4","outline_rule":"outlines-v3","page_budget":{"mode":"unlimited"},"parser_version":"0.64.0-dev.1","quantum_per_point":100,"raster_dpi":{"mode":"not_emitted"},"reading_order_rule":"gutter-columns-v4","right_to_left_rule":"right-to-left-lines-v1","struct_tree_rule":"struct-tree-v2","table_detection":{"charts":"bar-labels-v3","lines":"point-labels-v1","ruled":"ruled-rects-v12","stroke_ruled":"stroke-ruled-v1","tagged":"tagged-tables-v1","tracks":"whitespace-tracks-v14","unruled":"unruled-align-v1"},"text_box_rule":"advance-over-font-envelope-v1","text_code_rule":"declared-font-codes-v4","unicode_data_version":"std-unicode-16.0.0","verifier":{"mode":"not_pinned"},"xref_repair":{"mode":"pad-19-to-20-v1"}}"#, "the v0 profile changed. Expected causes: a crate version bump (parser_version is \ part of identity, so a new build IS a new profile — that is by design), or a new \ field. Update this vector and say why in the commit. Unexpected cause: something \ @@ -4634,11 +4699,19 @@ mod tests { set in. Nothing else.\n\n\ Moved for decision #64, `sha256:dcfd6050…` -> `sha256:3e5e6b07…`: \ `heading_inference_rule` `type-size-v6` -> `-v7`, which reads a run set a weight \ - heavier than its page's body as bold. Nothing else." + heavier than its page's body as bold. Nothing else.\n\n\ + Moved for decision #66, `sha256:3e5e6b07…` -> `sha256:3976f966…`: \ + `table_detection.charts` `bar-labels-v2` -> `-v3`, which names the one series that \ + prints numbers by the title over its chart, and the new `table_detection.lines`, \ + `point-labels-v1`, which reads the numbers a line chart prints at its points back \ + as its table. The office profiles name it `not-run-for-this-format`. Nothing else.\n\n\ + Moved for decision #67, `sha256:3976f966…` -> `sha256:e0ffb37d…`: \ + `layout_unit_rule` `line-units-v4` -> `-v5`, which reads each line where the reading \ + order reads it and its words where the page set them apart. Nothing else." ); assert_eq!( Profile::default().profile_sha256().unwrap().to_string(), - "sha256:3e5e6b071bcd48fe56258799c7a7e7b519579fdadd1049f7c4717182d11fd3aa" + "sha256:e0ffb37d1a2f8d95af4ac18a4eec88d651c4b0182bc7527858a38601b577b9e9" ); } @@ -4704,6 +4777,12 @@ mod tests { (TABLE_DETECTION_STROKE_V1, TABLE_DETECTION_CHARTS_V2), (TABLE_DETECTION_TAGGED_V1, TABLE_DETECTION_CHARTS_V2), (TABLE_DETECTION_TRACKS_V14, TABLE_DETECTION_CHARTS_V2), + (TABLE_DETECTION_V12, TABLE_DETECTION_LINES_V1), + (TABLE_DETECTION_UNRULED_V1, TABLE_DETECTION_LINES_V1), + (TABLE_DETECTION_STROKE_V1, TABLE_DETECTION_LINES_V1), + (TABLE_DETECTION_TAGGED_V1, TABLE_DETECTION_LINES_V1), + (TABLE_DETECTION_TRACKS_V14, TABLE_DETECTION_LINES_V1), + (TABLE_DETECTION_CHARTS_V3, TABLE_DETECTION_LINES_V1), ] { assert_ne!( a, b, @@ -4711,11 +4790,11 @@ mod tests { ); } - // All six appear on the wire, under their own keys. + // All seven appear on the wire, under their own keys. let s = String::from_utf8(base.canonical_bytes().unwrap()).unwrap(); assert!( s.contains( - r#""table_detection":{"charts":"bar-labels-v2","ruled":"ruled-rects-v12","stroke_ruled":"stroke-ruled-v1","tagged":"tagged-tables-v1","tracks":"whitespace-tracks-v14","unruled":"unruled-align-v1"}"# + r#""table_detection":{"charts":"bar-labels-v3","lines":"point-labels-v1","ruled":"ruled-rects-v12","stroke_ruled":"stroke-ruled-v1","tagged":"tagged-tables-v1","tracks":"whitespace-tracks-v14","unruled":"unruled-align-v1"}"# ), "{s}" ); @@ -4837,7 +4916,14 @@ mod tests { "a pre-#42 profile must not silently acquire the chart rule" ); - let good = r#"{"ruled":"ruled-rects-v12","unruled":"unruled-align-v1","stroke_ruled":"stroke-ruled-v1","tagged":"tagged-tables-v1","tracks":"whitespace-tracks-v14","charts":"bar-labels-v2"}"#; + // And a profile from before decision #66 must not acquire the line rule. + let pre_lines = r#"{"ruled":"ruled-rects-v12","unruled":"unruled-align-v1","stroke_ruled":"stroke-ruled-v1","tagged":"tagged-tables-v1","tracks":"whitespace-tracks-v14","charts":"bar-labels-v3"}"#; + assert!( + serde_json::from_str::(pre_lines).is_err(), + "a pre-#66 profile must not silently acquire the line rule" + ); + + let good = r#"{"ruled":"ruled-rects-v12","unruled":"unruled-align-v1","stroke_ruled":"stroke-ruled-v1","tagged":"tagged-tables-v1","tracks":"whitespace-tracks-v14","charts":"bar-labels-v3","lines":"point-labels-v1"}"#; assert_eq!( serde_json::from_str::(good).unwrap(), TableDetection::default() diff --git a/crates/ethos-parser-core/src/representation.rs b/crates/ethos-parser-core/src/representation.rs index 227b05d..6130d07 100644 --- a/crates/ethos-parser-core/src/representation.rs +++ b/crates/ethos-parser-core/src/representation.rs @@ -1613,7 +1613,10 @@ pub struct TextRunAttributes { /// and value can be two units — and since `line-units-v3` before a body line whose first word /// would have fit twice over after the line above, so a contents list is a unit per entry. Since /// `line-units-v4` a script — a subscript, a superscript, a footnote mark, on a baseline of its - /// own — is part of the line it is set in. + /// own — is part of the line it is set in. Since `line-units-v5` a line is the pieces of one + /// baseline that follow each other in reading order, so a baseline running across columns the + /// reading order reads apart is a line per column, and its text has a space where its words + /// stand apart with none drawn, so a bullet set apart by its position opens a list item. /// /// **Where, never what**, as [`Self::block`] is: a unit says these lines read as one piece of /// text, not that the author wrote a paragraph, and no role is read from it. Absent on a diff --git a/crates/ethos-parser-pdf/src/charts.rs b/crates/ethos-parser-pdf/src/charts.rs index 2a39617..124d2f6 100644 --- a/crates/ethos-parser-pdf/src/charts.rs +++ b/crates/ethos-parser-pdf/src/charts.rs @@ -91,31 +91,31 @@ pub(crate) struct Bar { /// Runs joined along a baseline: what a chart prints as one label. #[derive(Debug)] -struct Label { +pub(crate) struct Label { /// Its inked runs left to right, then the spaces drawn between them. - runs: Vec, - x0: i64, - y0: i64, - x1: i64, - y1: i64, + pub(crate) runs: Vec, + pub(crate) x0: i64, + pub(crate) y0: i64, + pub(crate) x1: i64, + pub(crate) y1: i64, /// The line's height: the median height of its runs' boxes. - h: i64, + pub(crate) h: i64, /// The baseline. - base: i64, + pub(crate) base: i64, /// What it reads as: the runs' text, a space between two that stand apart with none drawn. - text: String, + pub(crate) text: String, } impl Label { - fn cx(&self) -> i64 { + pub(crate) fn cx(&self) -> i64 { (self.x0 + self.x1).div_euclid(2) } - fn cy(&self) -> i64 { + pub(crate) fn cy(&self) -> i64 { (self.y0 + self.y1).div_euclid(2) } - fn rect(&self) -> QuantRect { + pub(crate) fn rect(&self) -> QuantRect { QuantRect { x0: self.x0, y0: self.y0, @@ -233,6 +233,18 @@ pub(crate) fn detect( Ok(tables) } +/// The bars of every chart the rule finds on a page, whether or not it reads a table from them +/// (`point-labels-v1`): a number set on a bar is the bar's, never a line's. +pub(crate) fn chart_bars(bars: &[Bar], page_area: i128) -> Vec { + let ink = ink(bars, page_area); + [Axis::Vertical, Axis::Horizontal] + .into_iter() + .flat_map(|axis| charts(&ink, axis)) + .flat_map(|c| c.groups.into_iter().flatten().flat_map(|col| col.segs)) + .map(|s| ink[s].rect) + .collect() +} + /// The labels a table reads: headers, categories, values. fn cells_of(f: &Found) -> Vec { let mut out: Vec = f.headers.iter().flatten().copied().collect(); @@ -245,7 +257,7 @@ fn cells_of(f: &Found) -> Vec { /// The page's labels: upright runs no other rule's table holds, by line, joined wherever less than /// half the line's height separates one run's ink from the next. -fn labels(runs: &[TrackRun<'_>]) -> Vec