Kansas munging improvements

sfsinger19103 · sfsinger19103 · commit 548c7249b42e · 2021-09-21T22:03:15.000-07:00
diff --git a/src/electiondata/munge/__init__.py b/src/electiondata/munge/__init__.py
@@ -624,6 +624,8 @@ def melt_to_one_count_column(
         if "in_count_headers" in p["munge_field_types"]:
             # split header_0 column into separate columns
             # # get header_rows
+            # TODO: the following throws PerformanceError for Kansas House of Representatives 2020g. Rather than
+            #  assigning values, need to use melted = pd.concat([melted, <new_columns>])
             melted[
                 [f"count_header_{idx}" for idx in p["count_header_row_numbers"]]
             ] = pd.DataFrame(melted["header_0"].str.split(";:;", expand=True).values)[
diff --git a/src/ini_files_for_results/Kansas/ks_20g_ks_house_official.ini b/src/ini_files_for_results/Kansas/ks_20g_ks_house_official.ini
@@ -1,6 +1,6 @@
 [election_results]
 results_file=Kansas/2020_General_Election_Kansas_House_of_Representatives_results_by_precinct.xlsx
-munger_list=ks_gen_main,ks_gen_johnson_count_from_B,ks_gen_shawnee_count_from_B,ks_gen_sedgwick,ks_gen_wyandotte_4_line_header_first_count_col_3
+munger_list=ks_gen_main,ks_gen_johnson_count_from_B,ks_gen_shawnee_count_from_B,ks_gen_sedgwick,ks_gen_wyandotte_4_line_header_first_count_col_3,ks_gen_wyandotte_3_line_header_first_count_col_3,ks_gen_wyandotte_4_line_header_first_count_col_3_merged_rows
 jurisdiction=Kansas
 election=2020 General
 results_short_name=ks_20g_kshouse
diff --git a/src/jurisdictions/Kansas/Candidate.txt b/src/jurisdictions/Kansas/Candidate.txt
@@ -125,15 +125,12 @@ Rick Kloos
 Rachel Willis
 Brenda S. Dietrich
 Anthony Hensley
-Under Votes
-Over Votes
 Laura McConwell
 Ethan Corson
 Diana Whittington
 Cindy Holscher
 Vail Fruechting
 Ty Masterson
-Total Votes Cast
 Timothy Don Fry II
 Mary Ware
 Dan Kerschen
@@ -356,3 +353,6 @@ Vic (T-Bone) Miller
 Vicki Schmidt
 Virgil Weigel
 Wendy Bingesser
+Jordan Michael Mackey
+Greg Conchola
+Rick Parsons
diff --git a/src/jurisdictions/Kansas/dictionary.txt b/src/jurisdictions/Kansas/dictionary.txt
@@ -167,7 +167,6 @@ Candidate	Molly Baumgardner	Baumgardner, Molly
 Candidate	Monica Murnan	Murnan, Monica
 Candidate	Nancy J. Ingle	Ingle, Nancy J.
 Candidate	Other	Other
-Candidate	Over Votes	Over Votes
 Candidate	Pat Pettey	Pat Pettey
 Candidate	Pat Proctor	Proctor, Pat
 Candidate	Patrick Penn	Penn, Patrick
@@ -224,13 +223,11 @@ Candidate	Todd Maddox	Maddox, Todd
 Candidate	Tom Hawk	Hawk, Tom
 Candidate	Tom Holland	Holland, Tom
 Candidate	Tory Marie Arnberger	Arnberger, Tory Marie
-Candidate	Total Votes Cast	Total Votes Cast
 Candidate	Tracey Mann	Mann, Tracey
 Candidate	Trevor Jacobs	Jacobs, Trevor
 Candidate	Troy L. Waymaster	Waymaster, Troy L.
 Candidate	Ty Masterson	Masterson, Ty
 Candidate	Ty Masterson	Ty Masterson
-Candidate	Under Votes	Under Votes
 Candidate	Vail Fruechting	Vail Fruechting
 Candidate	Virgil Peck	Peck, Virgil
 Candidate	W. Michael Shimeall	Shimeall, W. Michael
@@ -6761,3 +6758,6 @@ ReportingUnit	Kansas;Wilson County	Kansas;Wilson
 ReportingUnit	Kansas;Woodson County	Kansas;Woodson
 ReportingUnit	Kansas;Wyandotte County	Kansas;Wyandotte
 CandidateContest	KS Attorney General	KS;Attorney General;statewide
+Candidate	Jordan Michael Mackey	Jordan Michael Mackey
+Candidate	Greg Conchola	Greg Conchola
+Candidate	Rick Parsons	Rick Parsons
diff --git a/src/mungers/ks_gen_johnson_count_from_B.munger b/src/mungers/ks_gen_johnson_count_from_B.munger
@@ -49,14 +49,11 @@ CandidateContest=<count_header_0>
 Party=<count_header_2>
 
 
-
-
-
-
 # Values to ignore (optional) #
 [ignore]
 ## E.g: Candidate=Total Votes Cast,Registered Voters ##
 ReportingUnit=JOHNSON;COUNTY TOTALS,Johnson;COUNTY TOTALS
+Candidate=Write-in,Under Votes,Over Votes
 
 # Lookup formula sections #
 ## Required when foreign keys are used in munge formulas and    ##
diff --git a/src/mungers/ks_gen_sedgwick.munger b/src/mungers/ks_gen_sedgwick.munger
@@ -62,7 +62,7 @@ Party={<count_header_3>,^(\w\w\w) .*$}
 # Values to ignore (optional) #
 [ignore]
 ## E.g: Candidate=Total Votes Cast,Registered Voters ##
-Candidate=Write-in Totals,Totals
+Candidate=Write-in Totals,Totals,Total Votes Cast
 ReportingUnit=SEDGWICK;Totals,Sedgwick;Totals
 
 # Lookup formula sections #
diff --git a/src/mungers/ks_gen_shawnee.munger b/src/mungers/ks_gen_shawnee.munger
@@ -62,7 +62,7 @@ Party={<count_header_2>,^(\w\w\w) .*$}
 # Values to ignore (optional) #
 [ignore]
 ## E.g: Candidate=Total Votes Cast,Registered Voters ##
-Candidate=Write-in Totals
+Candidate=Write-in Totals,Write-in
 
 # Lookup formula sections #
 ## Required when foreign keys are used in munge formulas and    ##
diff --git a/src/mungers/ks_gen_wyandotte_3_line_header_first_count_col_3.munger b/src/mungers/ks_gen_wyandotte_3_line_header_first_count_col_3.munger
@@ -0,0 +1,70 @@
+# Format parameters section (required) #
+[format]
+## Required format parameters:
+#### File type must be one of: excel,json-nested,xml,flat_text
+file_type=excel
+#### Counts are found in one way of: by_name,by_number
+count_location=by_number:3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24
+
+merged_cells=yes
+
+################################################
+## Sometimes required format parameters:
+#### for flat_text file type:
+flat_text_delimiter=
+#### if count_columns_specified is 'by_name':
+count_fields_by_name=
+#### if count_columns_specified is 'by_number':
+#### if 'in_count_headers' is in munge_strings
+#### (start numbering from first unskipped row):
+count_header_row_numbers=0,1,2
+#### if 'constant_over_file' is in munge_strings (NB: give value for each in .ini file):
+constant_over_file=CountItemType
+#### if file type is flat_text or excel and count_columns_specified is 'by_name'
+#### (start numbering from first unskipped row):
+count_field_name_row=
+#### if file type is flat_text or excel and not all rows are data:
+#### (start numbering from first unskipped row):
+noncount_header_row=0
+
+################################################
+## Optional format parameters:
+#### for any file type:
+thousands_separator=,
+encoding=
+
+#### for a flat_text or excel file type:
+###### if field names are not given in file
+#all_rows=data
+###### if there are multiple blocks of data per page, each with its own headers
+multi_block=yes
+
+#### for excel file type:
+sheets_to_read_names=Wyandotte,WYANDOTTE
+sheets_to_read_numbers=
+sheets_to_skip_names=
+
+#### for xml file type
+nesting_tags=
+
+# Munge formula sections (required if in munge_strings list) #
+[munge formulas]
+ReportingUnit=<sheet_name>;<column_1>
+Candidate={<count_header_2>,^(?:\w\w\w |)(.*)$}
+CandidateContest=<count_header_0>
+Party={<count_header_2>,^(\w\w\w) .*$}
+
+
+
+# Values to ignore (optional) #
+[ignore]
+## E.g: Candidate=Total Votes Cast,Registered Voters ##
+ReportingUnit=WYANDOTTE;Totals,Wyandotte;Totals,WYANDOTTE;
+Candidate=Write-in Totals
+
+# Lookup formula sections #
+## Required when foreign keys are used in munge formulas and    ##
+## must be looked up in another table.                          ##
+## See mi_gen18.munger for example                              ##
+##################################################################
+
diff --git a/src/mungers/ks_gen_wyandotte_4_line_header_first_count_col_3.munger b/src/mungers/ks_gen_wyandotte_4_line_header_first_count_col_3.munger
@@ -60,7 +60,7 @@ Party={<count_header_2>,^(\w\w\w) .*$}
 [ignore]
 ## E.g: Candidate=Total Votes Cast,Registered Voters ##
 ReportingUnit=WYANDOTTE;Totals,Wyandotte;Totals,WYANDOTTE;
-Candidate=Write-in Totals
+Candidate=Write-in Totals,Vote For 1
 
 # Lookup formula sections #
 ## Required when foreign keys are used in munge formulas and    ##
diff --git a/src/mungers/ks_gen_wyandotte_4_line_header_first_count_col_3_merged_rows.munger b/src/mungers/ks_gen_wyandotte_4_line_header_first_count_col_3_merged_rows.munger
@@ -0,0 +1,70 @@
+# Format parameters section (required) #
+[format]
+## Required format parameters:
+#### File type must be one of: excel,json-nested,xml,flat_text
+file_type=excel
+#### Counts are found in one way of: by_name,by_number
+count_location=by_number:3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24
+
+merged_cells=yes
+
+################################################
+## Sometimes required format parameters:
+#### for flat_text file type:
+flat_text_delimiter=
+#### if count_columns_specified is 'by_name':
+count_fields_by_name=
+#### if count_columns_specified is 'by_number':
+#### if 'in_count_headers' is in munge_strings
+#### (start numbering from first unskipped row):
+count_header_row_numbers=0,2,3
+#### if 'constant_over_file' is in munge_strings (NB: give value for each in .ini file):
+constant_over_file=CountItemType
+#### if file type is flat_text or excel and count_columns_specified is 'by_name'
+#### (start numbering from first unskipped row):
+count_field_name_row=
+#### if file type is flat_text or excel and not all rows are data:
+#### (start numbering from first unskipped row):
+noncount_header_row=0
+
+################################################
+## Optional format parameters:
+#### for any file type:
+thousands_separator=,
+encoding=
+
+#### for a flat_text or excel file type:
+###### if field names are not given in file
+#all_rows=data
+###### if there are multiple blocks of data per page, each with its own headers
+multi_block=yes
+
+#### for excel file type:
+sheets_to_read_names=Wyandotte,WYANDOTTE
+sheets_to_read_numbers=
+sheets_to_skip_names=
+
+#### for xml file type
+nesting_tags=
+
+# Munge formula sections (required if in munge_strings list) #
+[munge formulas]
+ReportingUnit=<sheet_name>;<column_1>
+Candidate={<count_header_3>,^(?:\w\w\w |)(.*)$}
+CandidateContest=<count_header_0>
+Party={<count_header_3>,^(\w\w\w) .*$}
+
+
+
+# Values to ignore (optional) #
+[ignore]
+## E.g: Candidate=Total Votes Cast,Registered Voters ##
+ReportingUnit=WYANDOTTE;Totals,Wyandotte;Totals,WYANDOTTE;
+Candidate=Write-in Totals
+
+# Lookup formula sections #
+## Required when foreign keys are used in munge formulas and    ##
+## must be looked up in another table.                          ##
+## See mi_gen18.munger for example                              ##
+##################################################################
+
diff --git a/src/reference_results/Kansas.tsv b/src/reference_results/Kansas.tsv
@@ -2,7 +2,7 @@ Jurisdiction	Election	Contest	ReportingUnit	VoteType	Count	Status	Source	Note
 Kansas	2020 General	US President (KS)	Kansas	total	1457960	official-final		
 Kansas	2020 General	US House KS District 3	Kansas	total	410418	official-final		
 Kansas	2020 General	KS Senate District 15	Kansas	total	23043	official-final		
-Kansas	2020 General	KS Senate District 6	Kansas	total	22538	official-final
+Kansas	2020 General	KS Senate District 6	Kansas	total	22583	official-final
 Kansas	2020 General	KS Senate District 35	Kansas	total	28779	official-final
 Kansas	2020 General	KS House District 2	Kansas	total	10874	official-final
 Kansas	2020 General	KS House District 32	Kansas	total	3780	official-final
diff --git a/tests/dataloading_tests/conftest.py b/tests/dataloading_tests/conftest.py
@@ -30,7 +30,7 @@ def test_data_url(request):
     return request.config.getoption("--test_data_url")
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="function")
 def dataloader(param_file):
     ts = datetime.datetime.now().strftime("%m%d_%H%M")
     dbname = f"test_{ts}"
diff --git a/tests/dataloading_tests/test_dataloading_by_ej.py b/tests/dataloading_tests/test_dataloading_by_ej.py
@@ -0,0 +1,41 @@
+import pytest
+import os
+from pathlib import Path
+
+
+def test_dataloader_exists(dataloader):
+    assert dataloader is not None, (
+        "Specify viable dataloader parameter file path with --param_file option to pytest"
+        "or correct default param_file (run_time.ini in same folder as test_dataloading.py)"
+    )
+
+
+def test_loading(dataloader, test_data_url, param_file):
+    dataloader.get_testing_data_from_git_repo(test_data_url)
+    successfully_loaded, failed_to_load, all_tests_passed, err = dataloader.load_all(
+        move_files=False,
+        rollup=True,
+    )
+    print(
+        f"successfully loaded:\n{successfully_loaded}\n\n"
+        f"failed to load:\n{failed_to_load}\n\n"
+        f"passed all tests?:\n{all_tests_passed}"
+    )
+    # for all election-jurisdiction pairs attempted, can't have any files failing to load, and must have
+    #  all tests passed.
+    # set of ej-pairs attempted cannot be null
+    attempted_pairs = successfully_loaded.keys()
+    assert (
+        attempted_pairs
+    ), "No loadable results files found; check run_time.ini or specified parameter file"
+
+    # all files loaded successfully
+    assert all(
+        [v == list() for v in failed_to_load.values()]
+    ), f"Not all files loaded successfully."
+
+    # all tests passed
+    assert all([v for v in all_tests_passed.values()]), (
+        "Some tests failed.  For more information "
+        f"see reports in {dataloader.d['reports_and_plots_dir']}"
+    )
diff --git a/tests/dataloading_tests/test_dataloading_multi.py b/tests/dataloading_tests/test_dataloading_multi.py
@@ -10,37 +10,6 @@ def test_dataloader_exists(dataloader):
     )
 
 
-def test_loading(dataloader, test_data_url, param_file):
-    dataloader.get_testing_data_from_git_repo(test_data_url)
-    successfully_loaded, failed_to_load, all_tests_passed, err = dataloader.load_all(
-        move_files=False,
-        rollup=True,
-    )
-    print(
-        f"successfully loaded:\n{successfully_loaded}\n\n"
-        f"failed to load:\n{failed_to_load}\n\n"
-        f"passed all tests?:\n{all_tests_passed}"
-    )
-    # for all election-jurisdiction pairs attempted, can't have any files failing to load, and must have
-    #  all tests passed.
-    # set of ej-pairs attempted cannot be null
-    attempted_pairs = successfully_loaded.keys()
-    assert (
-        attempted_pairs
-    ), "No loadable results files found; check run_time.ini or specified parameter file"
-
-    # all files loaded successfully
-    assert all(
-        [v == list() for v in failed_to_load.values()]
-    ), f"Not all files loaded successfully."
-
-    # all tests passed
-    assert all([v for v in all_tests_passed.values()]), (
-        "Some tests failed.  For more information "
-        f"see reports in {dataloader.d['reports_and_plots_dir']}"
-    )
-
-
 def test_multielection_loading(dataloader):
     tests_dir = Path(__file__).parents[1]
     reference_results = os.path.join(
