rename munge_routines to munge

Author: Eric Tsai

src/election_anomaly/__init__.py

@@ -1,6 +1,6 @@
 from election_anomaly import database as db
 from election_anomaly import user_interface as ui
-from election_anomaly import munge_routines as mr
+from election_anomaly import munge as m
 from sqlalchemy.orm import sessionmaker
 import datetime
 from pathlib import Path
@@ -469,17 +469,17 @@ def add_sub_county_rus_from_results_file(
 			return error
 
 		# add columns for county and sub_ru
-		wr, error = mr.add_column_from_formula(wr,county_formula, 'County_raw', error, suffix='_SOURCE')
-		wr, error = mr.add_column_from_formula(wr,sub_ru_formula, 'Sub_County_raw', error, suffix='_SOURCE')
+		wr, error = m.add_column_from_formula(wr,county_formula, 'County_raw', error, suffix='_SOURCE')
+		wr, error = m.add_column_from_formula(wr,sub_ru_formula, 'Sub_County_raw', error, suffix='_SOURCE')
 
 		# add column for county internal name
 		ru_dict_old = prep.get_element(self.d['jurisdiction_path'],'dictionary')
 		ru_dict_new = ru_dict_old[ru_dict_old.cdf_element=='ReportingUnit']
 		wr = wr.merge(ru_dict_new,how='left',left_on='County_raw',right_on='raw_identifier_value').rename(columns={'cdf_internal_name':'County_internal'})
 
 		# add required new columns
-		wr = mr.add_constant_column(wr,'ReportingUnitType',sub_ru_type)
-		wr = mr.add_constant_column(wr,'cdf_element','ReportingUnit')
+		wr = m.add_constant_column(wr,'ReportingUnitType',sub_ru_type)
+		wr = m.add_constant_column(wr,'cdf_element','ReportingUnit')
 		wr['Name'] = wr.apply(lambda x: f'{x["County_internal"]};{x["Sub_County_raw"]}',axis=1)
 		wr['raw_identifier_value'] = wr.apply(lambda x: f'{x["County_raw"]};{x["Sub_County_raw"]}',axis=1)
 
@@ -542,7 +542,7 @@ def add_elements_from_results_file(self, elements: iter, error: dict, results_fi
 		for element in elements:
 			name_field = db.get_name_field(element)
 			# append <element>_raw
-			wr, error = mr.add_munged_column(
+			wr, error = m.add_munged_column(
 				wr, mu, element, error, mode=mu.cdf_elements.loc[element, 'source'],
 				inplace=False)
 			if error:
@@ -584,7 +584,7 @@ def starter_dictionary(self,include_existing=True) -> str:
 		for element in elements:
 			w[element] = prep.get_element(self.d['jurisdiction_path'],element)
 			name_field = db.get_name_field(element)
-			w[element] = mr.add_constant_column(w[element],'cdf_element',element)
+			w[element] = m.add_constant_column(w[element],'cdf_element',element)
 			w[element].rename(columns={name_field:'cdf_internal_name'},inplace=True)
 			w[element]['raw_identifier_value'] = w[element]['cdf_internal_name']
 

src/election_anomaly/analyze/__init__.py

@@ -3,7 +3,7 @@
 
 import pandas as pd
 from election_anomaly import user_interface as ui
-from election_anomaly import munge_routines as mr
+from election_anomaly import munge as m
 import datetime
 import os
 import numpy as np
@@ -536,7 +536,7 @@ def create_candidate_contests(df, columns):
 	if contest_df.empty:
 		contest_df['contest_type'] = None
 	else:
-		contest_df = mr.add_constant_column(contest_df,'contest_type','Candidate')
+		contest_df = m.add_constant_column(contest_df,'contest_type','Candidate')
 	return contest_df
 
 
@@ -549,7 +549,7 @@ def create_ballot_measure_contests(df, columns):
 	if ballotmeasure_df.empty:
 		ballotmeasure_df['contest_type'] = None
 	else:
-		ballotmeasure_df = mr.add_constant_column(ballotmeasure_df,'contest_type','BallotMeasure')
+		ballotmeasure_df = m.add_constant_column(ballotmeasure_df,'contest_type','BallotMeasure')
 	return ballotmeasure_df
 
 
@@ -563,7 +563,7 @@ def create_contests(df, reporting_units, candidate_columns=None, ballotmeasure_c
 	contest_selection = pd.concat([c_df, bm_df])
 	contest_selection = contest_selection.merge(reporting_units,
 		how='left', left_on='ElectionDistrict_Id', right_index=True)
-	contest_selection = mr.enum_col_from_id_othertext(contest_selection,'ReportingUnitType',df['ReportingUnitType'])
+	contest_selection = m.enum_col_from_id_othertext(contest_selection,'ReportingUnitType',df['ReportingUnitType'])
 	contest_selection.rename(columns={'ReportingUnitType':'contest_district_type'},inplace=True)
 	return contest_selection
 
@@ -612,6 +612,6 @@ def create_vote_counts(df, ecsvcj, contest_selection, ru_children, sub_ru):
 	}
 	unsummed.rename(columns=rename, inplace=True)
 	# add columns with names
-	unsummed = mr.enum_col_from_id_othertext(unsummed,'CountItemType',df['CountItemType'],drop_old=False)
+	unsummed = m.enum_col_from_id_othertext(unsummed,'CountItemType',df['CountItemType'],drop_old=False)
 	unsummed = unsummed.merge(contest_selection,how='left',on=['Selection_Id','Contest_Id'])
 	return unsummed

src/election_anomaly/database/__init__.py

@@ -15,7 +15,7 @@
 from election_anomaly import user_interface as ui
 from configparser import MissingSectionHeaderError
 import pandas as pd
-from election_anomaly import munge_routines as mr
+from election_anomaly import munge as m
 import re
 from election_anomaly.database import create_cdf_db as db_cdf
 import os
@@ -314,7 +314,7 @@ def insert_to_cdf_db(engine, df, element, sep='\t', encoding='iso-8859-1', times
 	it must be specified in <timestamp>; <df> must have columns matching <element>, except Id and <timestamp> if any"""
 
 	# initialize connection and cursor
-	working = mr.generic_clean(df)
+	working = m.generic_clean(df)
 	connection = engine.raw_connection()
 	cursor = connection.cursor()
 
@@ -363,7 +363,7 @@ def insert_to_cdf_db(engine, df, element, sep='\t', encoding='iso-8859-1', times
 
 	# add any missing columns needed for temp table to working
 	for c in temp_only_cols:
-		working = mr.add_constant_column(working,c,None)
+		working = m.add_constant_column(working,c,None)
 	working[temp_columns].drop_duplicates().to_csv(
 		output, sep=sep, header=False, encoding=encoding, index=False, quoting=csv.QUOTE_MINIMAL)
 	# set current position for the StringIO object to the beginning of the string
@@ -436,7 +436,7 @@ def append_id_to_dframe(engine: sqlalchemy.engine, df: pd.DataFrame, table, col_
 	df_cols = list(col_map.keys())
 
 	# create temp db table with info from df, without index
-	df = mr.generic_clean(df)
+	df = m.generic_clean(df)
 	df[df_cols].fillna('').to_sql(temp_table, engine,index_label='dataframe_index')
 	# TODO fillna('') probably redundant
 
@@ -450,7 +450,7 @@ def append_id_to_dframe(engine: sqlalchemy.engine, df: pd.DataFrame, table, col_
 	q = sql.SQL("SELECT t.*, tt.dataframe_index FROM {tt} tt LEFT JOIN {t} t ON {on_clause}").format(
 		tt=sql.Identifier(temp_table),t=sql.Identifier(table),on_clause=on_clause
 	)
-	w = mr.generic_clean(pd.read_sql_query(q, connection).set_index('dataframe_index'))
+	w = m.generic_clean(pd.read_sql_query(q, connection).set_index('dataframe_index'))
 
 	# drop temp db table
 	q = sql.SQL("DROP TABLE {temp_table}").format(temp_table=sql.Identifier(temp_table))

src/election_anomaly/juris_and_munger/__init__.py

@@ -3,7 +3,7 @@
 from election_anomaly import database as db
 import pandas as pd
 from pandas.api.types import is_numeric_dtype
-from election_anomaly import munge_routines as mr
+from election_anomaly import munge as m
 from election_anomaly import user_interface as ui
 import re
 import numpy as np
@@ -22,7 +22,7 @@ def load_contests(self, engine, contest_type: str, error: dict) -> dict:
             .fillna('none or unknown')
 
         # add contest_type column
-        df = mr.add_constant_column(df,'contest_type',contest_type)
+        df = m.add_constant_column(df,'contest_type',contest_type)
 
         # add 'none or unknown' record
         df = add_none_or_unknown(df,contest_type=contest_type)
@@ -125,7 +125,7 @@ def get_aux_data(self, aux_data_dir, err, project_root=None) -> dict:
 
             # cast primary key(s) as int if possible, and set as (multi-)index
             primary_keys = self.aux_meta.loc[abbrev, 'primary_key'].split(',')
-            df = mr.cast_cols_as_int(df,primary_keys,error_msg=f'In dataframe for {abbrev}')
+            df = m.cast_cols_as_int(df,primary_keys,error_msg=f'In dataframe for {abbrev}')
             df.set_index(primary_keys, inplace=True)
 
             aux_data_dict[abbrev] = df
@@ -151,7 +151,7 @@ def check_against_self(self):
             problems.append(f'''At least one source in cdf_elements.txt is not recognized: {b_str} ''')
 
         # formulas have good syntax
-        bad_formula = [x for x in self.cdf_elements.raw_identifier_formula.unique() if not mr.good_syntax(x)]
+        bad_formula = [x for x in self.cdf_elements.raw_identifier_formula.unique() if not m.good_syntax(x)]
         if bad_formula:
             f_str = ','.join(bad_formula)
             problems.append(f'''At least one formula in cdf_elements.txt has bad syntax: {f_str} ''')
@@ -228,7 +228,7 @@ def read_munger_info_from_files(dir_path,project_root=None,aux_data_dir=None):
     # add column for list of fields used in formulas
     cdf_elements['fields'] = [[]]*cdf_elements.shape[0]
     for i,r in cdf_elements.iterrows():
-        text_field_list,last_text = mr.text_fragments_and_fields(cdf_elements.loc[i,'raw_identifier_formula'])
+        text_field_list,last_text = m.text_fragments_and_fields(cdf_elements.loc[i,'raw_identifier_formula'])
         cdf_elements.loc[i,'fields'] = [f for t,f in text_field_list]
 
     # read formatting info
@@ -500,7 +500,7 @@ def check_munger_file_contents(munger_name,project_root):
         problems.append(f'''At least one source in cdf_elements.txt is not recognized: {b_str} ''')
 
     # formulas have good syntax
-    bad_formula = [x for x in cdf_elements.raw_identifier_formula.unique() if not mr.good_syntax(x)]
+    bad_formula = [x for x in cdf_elements.raw_identifier_formula.unique() if not m.good_syntax(x)]
     if bad_formula:
         f_str = ','.join(bad_formula)
         problems.append(f'''At least one formula in cdf_elements.txt has bad syntax: {f_str} ''')
@@ -663,7 +663,7 @@ def load_juris_dframe_into_cdf(session,element,juris_path,project_root,error,loa
             cdf_e = pd.read_sql_table(e,session.bind)
             # for every instance of the enumeration in the current table, add id and othertype columns to the dataframe
             if e in df.columns:
-                df = mr.enum_col_to_id_othertext(df,e,cdf_e)
+                df = m.enum_col_to_id_othertext(df,e,cdf_e)
 
     # get Ids for any foreign key (or similar) in the table, e.g., Party_Id, etc.
     fk_file_path = os.path.join(

…ction_anomaly/munge_routines/__init__.py src/election_anomaly/munge/__init__.pysrc/election_anomaly/munge_routines/__init__.py renamed to src/election_anomaly/munge/__init__.py src/election_anomaly/munge_routines/__init__.py renamed to src/election_anomaly/munge/__init__.py

@@ -1,7 +1,6 @@
 # Routines to aid in preparing Jurisdiction and Munger files
 import pandas as pd
 import os
-from election_anomaly import munge_routines as mr
 from election_anomaly import user_interface as ui
 from election_anomaly import juris_and_munger as jm
 from pathlib import Path

src/election_anomaly/munge_routines/__pycache__/__init__.cpython-38.pyc

@@ -1,5 +1,5 @@
 from configparser import ConfigParser
-from election_anomaly import munge_routines as mr
+from election_anomaly import munge as m
 import pandas as pd
 from pandas.errors import ParserError, ParserWarning
 import numpy as np
@@ -178,7 +178,7 @@ def read_single_datafile(munger: jm.Munger, f_path: str, err: dict) -> [pd.DataF
 			e = f'Nothing read from datafile; file type {munger.file_type} may be inconsistent, or datafile may be empty.'
 			add_error(err,'format.txt',e)
 		else:
-			df = mr.generic_clean(df)
+			df = m.generic_clean(df)
 			err = jm.check_results_munger_compatibility(munger, df, err)
 		return [df, err]
 	except UnicodeDecodeError as ude:
@@ -200,7 +200,7 @@ def read_combine_results(mu: jm.Munger, results_file, project_root, err, aux_dat
 	if [k for k in err.keys() if err[k] != None]:
 		return pd.DataFrame(), err
 	else:
-		working = mr.cast_cols_as_int(working, mu.count_columns,mode='index')
+		working = m.cast_cols_as_int(working, mu.count_columns,mode='index')
 
 		# merge with auxiliary files (if any)
 		if aux_data_dir is not None:
@@ -209,7 +209,7 @@ def read_combine_results(mu: jm.Munger, results_file, project_root, err, aux_dat
 			for abbrev,r in mu.aux_meta.iterrows():
 				# cast foreign key columns of main results file as int if possible
 				foreign_key = r['foreign_key'].split(',')
-				working = mr.cast_cols_as_int(working,foreign_key)
+				working = m.cast_cols_as_int(working,foreign_key)
 				# rename columns
 				col_rename = {f'{c}':f'{abbrev}[{c}]' for c in aux_data[abbrev].columns}
 				# merge auxiliary info into <working>
@@ -253,13 +253,13 @@ def new_datafile(
 	count_columns_by_name = [raw.columns[x] for x in munger.count_columns]
 
 	try:
-		raw = mr.munge_clean(raw, munger)
+		raw = m.munge_clean(raw, munger)
 	except:
 		err['datafile_error'] = ['Cleaning of datafile failed. Results not loaded to database.']
 		return err
 
 	try:
-		err = mr.raw_elements_to_cdf(session,project_root,juris,munger,raw,count_columns_by_name,err,ids=results_info)
+		err = m.raw_elements_to_cdf(session,project_root,juris,munger,raw,count_columns_by_name,err,ids=results_info)
 	except Exception as exc:
 		e = f'Unspecified error during munging: {exc}\nResults not loaded to database.'
 		add_error(err,'datafile_error',e)