From 60f98cb7d30b8029247c10ad94c47d4b61c43945 Mon Sep 17 00:00:00 2001 From: Richard Date: Sat, 11 Jul 2026 14:00:11 -0400 Subject: [PATCH 1/3] add advanced dictionary test --- .../sqllogictest/test_files/dictionary.slt | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/datafusion/sqllogictest/test_files/dictionary.slt b/datafusion/sqllogictest/test_files/dictionary.slt index 105523ab5090e..9932ed2c2e126 100644 --- a/datafusion/sqllogictest/test_files/dictionary.slt +++ b/datafusion/sqllogictest/test_files/dictionary.slt @@ -632,4 +632,61 @@ south 2 statement ok DROP TABLE dict_count_distinct; +# overlapping values with different key assignments across batches +query TI rowsort +WITH + first_batch AS ( + SELECT arrow_cast(column1, 'Dictionary(Int32, Utf8)') AS region + FROM (VALUES ('west'), ('east'), ('west'), (NULL)) AS t(column1) + ), + second_batch AS ( + SELECT arrow_cast(column1, 'Dictionary(Int16, LargeUtf8)') AS region + FROM (VALUES ('east'), ('west'), ('north'), (NULL)) AS t(column1) + ) +SELECT region, count(*) +FROM (SELECT region FROM first_batch UNION ALL SELECT region FROM second_batch) +GROUP BY region; +---- +NULL 2 +east 2 +north 1 +west 3 + +# different logical values must not merge even when their dictionary key numbers coincide +query TI rowsort +WITH + first_batch AS ( + SELECT arrow_cast(column1, 'Dictionary(Int8, LargeUtf8)') AS status + FROM (VALUES ('active'), ('inactive'), (NULL)) AS t(column1) + ), + second_batch AS ( + SELECT arrow_cast(column1, 'Dictionary(Int16, Utf8)') AS status + FROM (VALUES ('inactive'), ('active'), (NULL)) AS t(column1) + ) +SELECT status, count(*) +FROM (SELECT status FROM first_batch UNION ALL SELECT status FROM second_batch) +GROUP BY status; +---- +NULL 2 +active 2 +inactive 2 +# value present in one batch's dictionary but absent from the other +query TI rowsort +WITH + first_batch AS ( + SELECT arrow_cast(column1, 'Dictionary(Int32, Utf8)') AS sensor + FROM (VALUES ('temperature'), ('humidity'), ('temperature'), (NULL)) AS t(column1) + ), + second_batch AS ( + SELECT arrow_cast(column1, 'Dictionary(Int8, LargeUtf8)') AS sensor + FROM (VALUES ('temperature'), ('pressure'), (NULL)) AS t(column1) + ) +SELECT sensor, count(*) +FROM (SELECT sensor FROM first_batch UNION ALL SELECT sensor FROM second_batch) +GROUP BY sensor; +---- +NULL 2 +humidity 1 +pressure 1 +temperature 3 From fb5f27620b99d00c230f4e347c7e81dffcd2f9a2 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 14 Jul 2026 11:12:35 -0400 Subject: [PATCH 2/3] PR revisions --- .../sqllogictest/test_files/dictionary.slt | 44 ++----------------- 1 file changed, 3 insertions(+), 41 deletions(-) diff --git a/datafusion/sqllogictest/test_files/dictionary.slt b/datafusion/sqllogictest/test_files/dictionary.slt index 9932ed2c2e126..751d4da877384 100644 --- a/datafusion/sqllogictest/test_files/dictionary.slt +++ b/datafusion/sqllogictest/test_files/dictionary.slt @@ -632,7 +632,8 @@ south 2 statement ok DROP TABLE dict_count_distinct; -# overlapping values with different key assignments across batches +# same dictionary type but value order differs across batches so key ids refer to different strings; +# grouping must use the logical value, not the raw key id query TI rowsort WITH first_batch AS ( @@ -640,7 +641,7 @@ WITH FROM (VALUES ('west'), ('east'), ('west'), (NULL)) AS t(column1) ), second_batch AS ( - SELECT arrow_cast(column1, 'Dictionary(Int16, LargeUtf8)') AS region + SELECT arrow_cast(column1, 'Dictionary(Int32, Utf8)') AS region FROM (VALUES ('east'), ('west'), ('north'), (NULL)) AS t(column1) ) SELECT region, count(*) @@ -651,42 +652,3 @@ NULL 2 east 2 north 1 west 3 - -# different logical values must not merge even when their dictionary key numbers coincide -query TI rowsort -WITH - first_batch AS ( - SELECT arrow_cast(column1, 'Dictionary(Int8, LargeUtf8)') AS status - FROM (VALUES ('active'), ('inactive'), (NULL)) AS t(column1) - ), - second_batch AS ( - SELECT arrow_cast(column1, 'Dictionary(Int16, Utf8)') AS status - FROM (VALUES ('inactive'), ('active'), (NULL)) AS t(column1) - ) -SELECT status, count(*) -FROM (SELECT status FROM first_batch UNION ALL SELECT status FROM second_batch) -GROUP BY status; ----- -NULL 2 -active 2 -inactive 2 - -# value present in one batch's dictionary but absent from the other -query TI rowsort -WITH - first_batch AS ( - SELECT arrow_cast(column1, 'Dictionary(Int32, Utf8)') AS sensor - FROM (VALUES ('temperature'), ('humidity'), ('temperature'), (NULL)) AS t(column1) - ), - second_batch AS ( - SELECT arrow_cast(column1, 'Dictionary(Int8, LargeUtf8)') AS sensor - FROM (VALUES ('temperature'), ('pressure'), (NULL)) AS t(column1) - ) -SELECT sensor, count(*) -FROM (SELECT sensor FROM first_batch UNION ALL SELECT sensor FROM second_batch) -GROUP BY sensor; ----- -NULL 2 -humidity 1 -pressure 1 -temperature 3 From 4ca86a1075c6b82447dfc0b677dd256203e644e3 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 14 Jul 2026 11:20:39 -0400 Subject: [PATCH 3/3] more obvious test --- datafusion/sqllogictest/test_files/dictionary.slt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/datafusion/sqllogictest/test_files/dictionary.slt b/datafusion/sqllogictest/test_files/dictionary.slt index 751d4da877384..f314254955824 100644 --- a/datafusion/sqllogictest/test_files/dictionary.slt +++ b/datafusion/sqllogictest/test_files/dictionary.slt @@ -638,17 +638,16 @@ query TI rowsort WITH first_batch AS ( SELECT arrow_cast(column1, 'Dictionary(Int32, Utf8)') AS region - FROM (VALUES ('west'), ('east'), ('west'), (NULL)) AS t(column1) + FROM (VALUES ('west'), ('west'), ('west'), ('east'), (NULL)) AS t(column1) ), second_batch AS ( SELECT arrow_cast(column1, 'Dictionary(Int32, Utf8)') AS region - FROM (VALUES ('east'), ('west'), ('north'), (NULL)) AS t(column1) + FROM (VALUES ('east'), ('east'), ('east'), ('west'), (NULL)) AS t(column1) ) SELECT region, count(*) FROM (SELECT region FROM first_batch UNION ALL SELECT region FROM second_batch) GROUP BY region; ---- NULL 2 -east 2 -north 1 -west 3 +east 4 +west 4