First test for create-data without intermediate file

Author: Tim Band

datafaker/base.py

@@ -22,24 +22,6 @@
 )
 
 
-class TableGenerator(ABC):
-    """Abstract base class for table generator classes."""
-
-    num_rows_per_pass: int = 1
-
-    @abstractmethod
-    def __call__(self, dst_db_conn: Connection, metadata: MetaData) -> dict[str, Any]:
-        """Return, as a dictionary, a new row for the table that we are generating.
-
-        The only argument, `dst_db_conn`, should be a database connection to the
-        database to which the data is being written. Most generators won't use it, but
-        some do, and thus it's required by the interface.
-
-        The return value should be a dictionary with column names as strings for keys,
-        and the values being the values for the new row.
-        """
-
-
 @dataclass
 class FileUploader:
     """For uploading data files."""

datafaker/create.py

@@ -1,7 +1,6 @@
 """Functions and classes to create and populate the target database."""
-import pathlib
 from collections import Counter
-from types import ModuleType
+from pathlib import Path
 from typing import Any, Generator, Iterable, Iterator, Mapping, Sequence, Tuple
 
 from sqlalchemy import Connection, insert, inspect
@@ -10,10 +9,19 @@
 from sqlalchemy.orm import Session
 from sqlalchemy.schema import CreateColumn, CreateSchema, CreateTable, MetaData, Table
 
-from datafaker.base import FileUploader, TableGenerator
+from datafaker.base import FileUploader
+from datafaker.make import get_generation_info
+from datafaker.populate import (
+    TableGenerator,
+    get_symbols,
+    get_table_generator_dict,
+    get_story_generator_list,
+    get_vocab_dict,
+)
 from datafaker.settings import get_destination_dsn, get_destination_schema, get_settings
 from datafaker.utils import (
     create_db_engine_dst,
+    get_property,
     get_sync_engine,
     get_vocabulary_table_names,
     logger,
@@ -92,7 +100,7 @@ def create_db_vocab(
     metadata: MetaData,
     meta_dict: dict[str, Any],
     config: Mapping,
-    base_path: pathlib.Path = pathlib.Path("."),
+    base_path: Path = Path("."),
 ) -> list[str]:
     """
     Load vocabulary tables from files.
@@ -140,14 +148,16 @@ def create_db_vocab(
 
 def create_db_data(
     sorted_tables: Sequence[Table],
-    df_module: ModuleType,
+    config: Mapping[str, Any],
+    src_stats_filename: Path | None,
     num_passes: int,
     metadata: MetaData,
 ) -> RowCounts:
     """Connect to a database and populate it with data."""
     return create_db_data_into(
         sorted_tables,
-        df_module,
+        config,
+        src_stats_filename,
         num_passes,
         get_destination_dsn(),
         get_destination_schema(),
@@ -158,7 +168,8 @@ def create_db_data(
 # pylint: disable=too-many-arguments too-many-positional-arguments
 def create_db_data_into(
     sorted_tables: Sequence[Table],
-    df_module: ModuleType,
+    config: Mapping[str, Any],
+    src_stats_filename: Path | None,
     num_passes: int,
     db_dsn: str,
     schema_name: str | None,
@@ -176,17 +187,31 @@ def create_db_data_into(
     :param num_passes: Number of passes to perform.
     :param db_dsn: Connection string for the destination database.
     :param schema_name: Destination schema name.
+    :param metadata: Destination database metadata.
     """
     dst_engine = get_sync_engine(create_db_engine_dst(db_dsn, schema_name=schema_name))
-
+    gen_info = get_generation_info(metadata, config, Path("orm.blah"), Path("config.blah"), src_stats_filename)
     row_counts: Counter[str] = Counter()
     with dst_engine.connect() as dst_conn:
+        context = get_symbols(
+            gen_info.row_generator_module_name,
+            gen_info.story_generator_module_name,
+            get_property(config, "object_instantiation", dict, {}),
+            gen_info.src_stats_filename,
+            dst_conn,
+            metadata,
+        )
         for _ in range(num_passes):
             row_counts += populate(
                 dst_conn,
                 sorted_tables,
-                df_module.table_generator_dict,
-                df_module.story_generator_list,
+                get_table_generator_dict(
+                    dst_conn,
+                    gen_info.tables,
+                    gen_info.max_unique_constraint_tries,
+                    context,
+                ),
+                get_story_generator_list(gen_info.story_generators, context),
                 metadata,
             )
     dst_engine.dispose()
@@ -336,7 +361,7 @@ def populate(
         try:
             with dst_conn.begin():
                 for _ in range(table_generator.num_rows_per_pass):
-                    stmt = insert(table).values(table_generator(dst_conn, metadata))
+                    stmt = insert(table).values(table_generator(dst_conn))
                     dst_conn.execute(stmt)
                     row_counts[table.name] = row_counts.get(table.name, 0) + 1
                 dst_conn.commit()

datafaker/main.py

@@ -147,15 +147,19 @@ def create_data(
         help="The name of the ORM yaml file",
         dir_okay=False,
     ),
-    df_file: str = Option(
-        DF_FILENAME,
-        help="The name of the generators file. Must be in the current working directory.",
-        dir_okay=False,
-    ),
     config_file: Optional[Path] = Option(
         CONFIG_FILENAME,
         help="The configuration file",
     ),
+    stats_file: Optional[Path] = Option(
+        None,
+        help=(
+            "Statistics file (output of make-stats); default is src-stats.yaml if the "
+            "config file references SRC_STATS, or None otherwise."
+        ),
+        show_default=False,
+        dir_okay=False,
+    ),
     num_passes: int = Option(1, help="Number of passes (rows or stories) to make"),
 ) -> None:
     """Populate the schema in the target directory with synthetic data.
@@ -179,11 +183,11 @@ def create_data(
     logger.debug("Creating data.")
     config = read_config_file(config_file) if config_file is not None else {}
     orm_metadata = load_metadata_for_output(orm_file, config)
-    df_module = import_file(df_file)
     try:
         row_counts = create_db_data(
             sorted_non_vocabulary_tables(orm_metadata, config),
-            df_module,
+            config,
+            stats_file,
             num_passes,
             orm_metadata,
         )

datafaker/make.py

@@ -66,7 +66,8 @@ class FunctionCall:
     """Contains the df.py content related function calls."""
 
     function_name: str
-    argument_values: list[str]
+    args: list[str]
+    kwargs: dict[str, str]
 
 
 @dataclass
@@ -83,7 +84,8 @@ class ColumnChoice:
     """Choose columns based on a random number in [0,1)."""
 
     function_name: str
-    argument_values: list[str]
+    args: list[str]
+    kwargs: dict[str, str]
 
 
 def make_column_choices(
@@ -100,7 +102,8 @@ def make_column_choices(
     return [
         ColumnChoice(
             function_name=mg["name"],
-            argument_values=[f"{k}={v}" for k, v in mg.get("kwargs", {}).items()],
+            args=mg.get("args", []),
+            kwargs=mg.get("kwargs", {}),
         )
         for mg in table_config.get("missingness_generators", [])
         if "name" in mg
@@ -168,12 +171,11 @@ def _get_function_call(
     if keyword_arguments is None:
         keyword_arguments = {}
 
-    argument_values: list[str] = [str(value) for value in positional_arguments]
-    argument_values += [
-        f"{key}={_render_value(value)}" for key, value in keyword_arguments.items()
-    ]
-
-    return FunctionCall(function_name=function_name, argument_values=argument_values)
+    return FunctionCall(
+        function_name=function_name,
+        args=positional_arguments,
+        kwargs=keyword_arguments,
+    )
 
 
 def _get_row_generator(
@@ -580,13 +582,29 @@ def make_vocabulary_tables(
         )
 
 
-def make_table_generators(  # pylint: disable=too-many-locals
+@dataclass
+class GenerationInfo:
+    """Information for the generation of all data."""
+    provider_imports: list[str]
+    orm_file_name: Path
+    config_file_name: Path
+    row_generator_module_name: str | None
+    story_generator_module_name: str | None
+    object_instantiation: dict[str, dict]
+    src_stats_filename: Path | None
+    tables: list[TableGeneratorInfo]
+    vocabulary_tables: list[VocabularyTableGeneratorInfo]
+    story_generators: list[StoryGeneratorInfo]
+    max_unique_constraint_tries: int | None
+
+
+def get_generation_info(  # pylint: disable=too-many-locals
     metadata: MetaData,
     config: Mapping,
     orm_filename: Path,
     config_filename: Path,
     src_stats_filename: Optional[Path],
-) -> str:
+) -> GenerationInfo:
     """
     Create datafaker generator classes.
 
@@ -605,10 +623,16 @@ def make_table_generators(  # pylint: disable=too-many-locals
 
     :return: A string that is a valid Python module, once written to file.
     """
-    row_generator_module_name: str = config.get("row_generators_module", None)
-    story_generator_module_name = config.get("story_generators_module", None)
-    object_instantiation: dict[str, dict] = config.get("object_instantiation", {})
-    tables_config = config.get("tables", {})
+    row_generator_module_name = get_property(
+        config, "row_generators_module", str | None, None
+    )
+    story_generator_module_name = get_property(
+        config, "story_generators_module", str | None, None
+    )
+    object_instantiation = get_property(
+        config, "object_instantiation", dict, {}
+    )
+    tables_config = get_property(config, "tables", dict, {})
 
     tables: list[TableGeneratorInfo] = []
     vocabulary_tables: list[VocabularyTableGeneratorInfo] = []
@@ -637,20 +661,47 @@ def make_table_generators(  # pylint: disable=too-many-locals
 
     story_generators = _get_story_generators(config)
 
-    max_unique_constraint_tries = config.get("max-unique-constraint-tries", None)
+    max_unique_constraint_tries = get_property(
+        config, "max-unique-constraint-tries", str | None, None
+    )
+    return GenerationInfo(
+        provider_imports=PROVIDER_IMPORTS,
+        orm_file_name=orm_filename,
+        config_file_name=config_filename,
+        row_generator_module_name=row_generator_module_name,
+        story_generator_module_name=story_generator_module_name,
+        object_instantiation=object_instantiation,
+        src_stats_filename=src_stats_filename,
+        tables=tables,
+        vocabulary_tables=vocabulary_tables,
+        story_generators=story_generators,
+        max_unique_constraint_tries=max_unique_constraint_tries,
+    )
+
+
+def make_table_generators(  # pylint: disable=too-many-locals
+    metadata: MetaData,
+    config: Mapping,
+    orm_filename: Path,
+    config_filename: Path,
+    src_stats_filename: Optional[Path],
+) -> str:
+    gi = get_generation_info(
+        metadata, config, orm_filename, config_filename, src_stats_filename
+    )
     return generate_df_content(
         {
-            "provider_imports": PROVIDER_IMPORTS,
-            "orm_file_name": orm_filename,
-            "config_file_name": config_filename,
-            "row_generator_module_name": row_generator_module_name,
-            "story_generator_module_name": story_generator_module_name,
-            "object_instantiation": object_instantiation,
-            "src_stats_filename": src_stats_filename,
-            "tables": tables,
-            "vocabulary_tables": vocabulary_tables,
-            "story_generators": story_generators,
-            "max_unique_constraint_tries": max_unique_constraint_tries,
+            "provider_imports": gi.provider_imports,
+            "orm_file_name": gi.orm_file_name,
+            "config_file_name": gi.config_file_name,
+            "row_generator_module_name": gi.row_generator_module_name,
+            "story_generator_module_name": gi.story_generator_module_name,
+            "object_instantiation": gi.object_instantiation,
+            "src_stats_filename": gi.src_stats_filename,
+            "tables": gi.tables,
+            "vocabulary_tables": gi.vocabulary_tables,
+            "story_generators": gi.story_generators,
+            "max_unique_constraint_tries": gi.max_unique_constraint_tries,
         }
     )