From 369d11faf977e8755ebabd7ac0a178d68f7e61f1 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Mon, 25 May 2026 11:50:50 +0000 Subject: [PATCH 1/9] chore: automated project metadata update --- FEATURES.md | 1 + mysqltuner.pl | 146 +++++++++++++++++++++++++++++--------------------- 2 files changed, 86 insertions(+), 61 deletions(-) diff --git a/FEATURES.md b/FEATURES.md index 77d156677..9aed2ad5e 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -11,6 +11,7 @@ Features list for option: --feature (dev only) * display_health_score * execute_system_command * execute_system_command; +* find_dominant_style * generate_auto_fix_snippets * historical_comparison * log_file_recommendations diff --git a/mysqltuner.pl b/mysqltuner.pl index 17fa27267..b5588d4d0 100755 --- a/mysqltuner.pl +++ b/mysqltuner.pl @@ -4257,7 +4257,7 @@ sub check_auth_plugins { mysql_version_ge( 9, 0 ) && ( $myvar{'version'} !~ /MariaDB/i ); my $is_mariadb = ( $myvar{'version'} =~ /MariaDB/i ); - my $insecure_count = 0; + my $insecure_count = 0; my $sha256_insecure_count = 0; foreach my $line (@mysqlstatlist) { my ( $user_host, $plugin ) = split( /\t/, $line ); @@ -4301,7 +4301,8 @@ sub check_auth_plugins { push @generalrec, $rec; } if ( $sha256_insecure_count > 0 ) { - push @generalrec, "Migrate to 'caching_sha2_password' for $sha256_insecure_count user(s) (using sha256_password)"; + push @generalrec, +"Migrate to 'caching_sha2_password' for $sha256_insecure_count user(s) (using sha256_password)"; } if ( $insecure_count == 0 && $sha256_insecure_count == 0 ) { @@ -8542,8 +8543,9 @@ sub mysql_table_structures { if ( $column ne 'id' && $column ne "${table}_id" ) { badprint "Table $schema.$table: Primary key '$column' does not follow 'id' or '${table}_id' naming convention"; - # push @generalrec, - # "Use 'id' or '${table}_id' for Primary Key naming in $schema.$table"; + + # push @generalrec, + # "Use 'id' or '${table}_id' for Primary Key naming in $schema.$table"; push @modeling, "Table $schema.$table: Primary key '$column' does not follow naming convention (id or ${table}_id)"; $pk_naming_issues_count++; @@ -8568,8 +8570,9 @@ sub mysql_table_structures { else { badprint "Table $schema.$table: Primary key '$column' is not a recommended surrogate key (BIGINT UNSIGNED AUTO_INCREMENT)"; - # push @generalrec, - # "Use BIGINT UNSIGNED AUTO_INCREMENT for Primary Keys in $schema.$table"; + + # push @generalrec, + # "Use BIGINT UNSIGNED AUTO_INCREMENT for Primary Keys in $schema.$table"; push @modeling, "Table $schema.$table: Primary key '$column' is not a recommended surrogate key (BIGINT UNSIGNED AUTO_INCREMENT)"; $bigint_pk_issues_count++; @@ -8578,10 +8581,12 @@ sub mysql_table_structures { } if ( $pk_naming_issues_count > 0 ) { - push @generalrec, "Use 'id' or '__id' for Primary Key naming in $pk_naming_issues_count table(s)"; + push @generalrec, +"Use 'id' or '_
_id' for Primary Key naming in $pk_naming_issues_count table(s)"; } if ( $bigint_pk_issues_count > 0 ) { - push @generalrec, "Use BIGINT UNSIGNED AUTO_INCREMENT for Primary Keys in $bigint_pk_issues_count table(s)"; + push @generalrec, +"Use BIGINT UNSIGNED AUTO_INCREMENT for Primary Keys in $bigint_pk_issues_count table(s)"; } # Large Tables (>1GB) without Secondary Indexes @@ -8753,16 +8758,18 @@ sub mysql_80_modeling_checks { if ( scalar(@genCols) == 0 ) { infoprint "Table $schema.$table: JSON column '$column' detected without Virtual Generated Columns for indexing"; - # push @generalrec, - # "Consider using Generated Columns to index frequently searched attributes in JSON column $schema.$table.$column"; + +# push @generalrec, +# "Consider using Generated Columns to index frequently searched attributes in JSON column $schema.$table.$column"; push @modeling, "Table $schema.$table: JSON column '$column' detected without Virtual Generated Columns for indexing"; $json_columns_without_virtual_count++; $modeling80Count++; } } - if ($json_columns_without_virtual_count > 0) { - push @generalrec, "Consider using Generated Columns to index frequently searched attributes in JSON column in $json_columns_without_virtual_count column(s)"; + if ( $json_columns_without_virtual_count > 0 ) { + push @generalrec, +"Consider using Generated Columns to index frequently searched attributes in JSON column in $json_columns_without_virtual_count column(s)"; } # Invisible Indexes (MySQL: IS_VISIBLE='NO', MariaDB: IGNORED='YES') @@ -8826,19 +8833,19 @@ sub get_compatible_styles { my ($name) = @_; return () unless defined $name && $name ne ''; my @styles; - if ($name =~ /^[a-z0-9]+(?:_[a-z0-9]+)*$/) { + if ( $name =~ /^[a-z0-9]+(?:_[a-z0-9]+)*$/ ) { push @styles, 'snake_case'; } - if ($name =~ /^[a-z0-9]+(?:[A-Z0-9][a-z0-9]*)*$/) { + if ( $name =~ /^[a-z0-9]+(?:[A-Z0-9][a-z0-9]*)*$/ ) { push @styles, 'camelCase'; } - if ($name =~ /^[A-Z0-9][a-z0-9]*(?:[A-Z0-9][a-z0-9]*)*$/) { + if ( $name =~ /^[A-Z0-9][a-z0-9]*(?:[A-Z0-9][a-z0-9]*)*$/ ) { push @styles, 'PascalCase'; } - if ($name =~ /^[a-z0-9]+(?:-[a-z0-9]+)*$/) { + if ( $name =~ /^[a-z0-9]+(?:-[a-z0-9]+)*$/ ) { push @styles, 'kebab-case'; } - if ($name =~ /^[A-Z0-9]+(?:_[A-Z0-9]+)*$/) { + if ( $name =~ /^[A-Z0-9]+(?:_[A-Z0-9]+)*$/ ) { push @styles, 'UPPER_SNAKE_CASE'; } return @styles; @@ -8853,12 +8860,14 @@ sub find_dominant_style { $style_counts{$style}++; } } - my $dominant = 'snake_case'; # Default fallback + my $dominant = 'snake_case'; # Default fallback my $max_count = 0; - foreach my $style (qw(snake_case camelCase PascalCase kebab-case UPPER_SNAKE_CASE)) { - if (($style_counts{$style} // 0) > $max_count) { + foreach my $style ( + qw(snake_case camelCase PascalCase kebab-case UPPER_SNAKE_CASE)) + { + if ( ( $style_counts{$style} // 0 ) > $max_count ) { $max_count = $style_counts{$style}; - $dominant = $style; + $dominant = $style; } } return $dominant; @@ -8867,19 +8876,19 @@ sub find_dominant_style { sub mysql_naming_conventions { subheaderprint "Naming conventions analysis"; - my $namingIssues = 0; + my $namingIssues = 0; my $plural_table_issues_count = 0; - my $table_style_issues_count = 0; - my $view_style_issues_count = 0; - my $index_style_issues_count = 0; + my $table_style_issues_count = 0; + my $view_style_issues_count = 0; + my $index_style_issues_count = 0; my $column_style_issues_count = 0; # Table Naming my @tables = select_array( "SELECT TABLE_SCHEMA, TABLE_NAME FROM information_schema.tables WHERE TABLE_TYPE = 'BASE TABLE' AND TABLE_SCHEMA NOT IN ('sys', 'mysql', 'performance_schema', 'information_schema')" ); - my @table_names = map { (split /\t/, $_)[1] // '' } @tables; - my $dominant_table_style = find_dominant_style(\@table_names); + my @table_names = map { ( split /\t/, $_ )[1] // '' } @tables; + my $dominant_table_style = find_dominant_style( \@table_names ); foreach my $t (@tables) { my ( $schema, $table ) = split /\t/, $t; @@ -8892,6 +8901,7 @@ sub mysql_naming_conventions { { badprint "Table $schema.$table: Plural name detected (prefer singular)"; + # push @generalrec, "Use singular names for table $schema.$table"; push @modeling, "Table $schema.$table: Plural name detected (prefer singular)"; @@ -8900,10 +8910,12 @@ sub mysql_naming_conventions { } # Casing check (detect CamelCase/PascalCase or other non-dominant) - my @compat = get_compatible_styles($table); + my @compat = get_compatible_styles($table); my $is_compatible = grep { $_ eq $dominant_table_style } @compat; - if (!$is_compatible) { - badprint "Table $schema.$table: Non-${dominant_table_style} name detected"; + if ( !$is_compatible ) { + badprint + "Table $schema.$table: Non-${dominant_table_style} name detected"; + # push @generalrec, "Use snake_case for table $schema.$table"; push @modeling, "Table $schema.$table: Non-${dominant_table_style} name detected"; @@ -8916,19 +8928,21 @@ sub mysql_naming_conventions { my @views = select_array( "SELECT TABLE_SCHEMA, TABLE_NAME FROM information_schema.tables WHERE TABLE_TYPE = 'VIEW' AND TABLE_SCHEMA NOT IN ('sys', 'mysql', 'performance_schema', 'information_schema')" ); - my @view_names = map { (split /\t/, $_)[1] // '' } @views; - my $dominant_view_style = find_dominant_style(\@view_names); + my @view_names = map { ( split /\t/, $_ )[1] // '' } @views; + my $dominant_view_style = find_dominant_style( \@view_names ); foreach my $v (@views) { my ( $schema, $view ) = split /\t/, $v; $schema //= ''; - $view //= ''; + $view //= ''; - my @compat = get_compatible_styles($view); + my @compat = get_compatible_styles($view); my $is_compatible = grep { $_ eq $dominant_view_style } @compat; - if (!$is_compatible) { - badprint "View $schema.$view: Non-${dominant_view_style} name detected"; - push @modeling, "View $schema.$view: Non-${dominant_view_style} name detected"; + if ( !$is_compatible ) { + badprint + "View $schema.$view: Non-${dominant_view_style} name detected"; + push @modeling, + "View $schema.$view: Non-${dominant_view_style} name detected"; $view_style_issues_count++; $namingIssues++; } @@ -8938,8 +8952,8 @@ sub mysql_naming_conventions { my @indexes = select_array( "SELECT DISTINCT TABLE_SCHEMA, TABLE_NAME, INDEX_NAME FROM information_schema.statistics WHERE INDEX_NAME != 'PRIMARY' AND TABLE_SCHEMA NOT IN ('sys', 'mysql', 'performance_schema', 'information_schema')" ); - my @index_names = map { (split /\t/, $_)[2] // '' } @indexes; - my $dominant_index_style = find_dominant_style(\@index_names); + my @index_names = map { ( split /\t/, $_ )[2] // '' } @indexes; + my $dominant_index_style = find_dominant_style( \@index_names ); foreach my $idx (@indexes) { my ( $schema, $table, $index ) = split /\t/, $idx; @@ -8947,11 +8961,13 @@ sub mysql_naming_conventions { $table //= ''; $index //= ''; - my @compat = get_compatible_styles($index); + my @compat = get_compatible_styles($index); my $is_compatible = grep { $_ eq $dominant_index_style } @compat; - if (!$is_compatible) { - badprint "Index $schema.$table.$index: Non-${dominant_index_style} name detected"; - push @modeling, "Index $schema.$table.$index: Non-${dominant_index_style} name detected"; + if ( !$is_compatible ) { + badprint +"Index $schema.$table.$index: Non-${dominant_index_style} name detected"; + push @modeling, +"Index $schema.$table.$index: Non-${dominant_index_style} name detected"; $index_style_issues_count++; $namingIssues++; } @@ -8961,8 +8977,8 @@ sub mysql_naming_conventions { my @columns = select_array( "SELECT TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME, DATA_TYPE FROM information_schema.columns WHERE TABLE_SCHEMA NOT IN ('sys', 'mysql', 'performance_schema', 'information_schema')" ); - my @column_names = map { (split /\t/, $_)[2] // '' } @columns; - my $dominant_column_style = find_dominant_style(\@column_names); + my @column_names = map { ( split /\t/, $_ )[2] // '' } @columns; + my $dominant_column_style = find_dominant_style( \@column_names ); foreach my $c (@columns) { my ( $schema, $table, $column, $datatype ) = split /\t/, $c; @@ -8972,15 +8988,16 @@ sub mysql_naming_conventions { $datatype //= ''; # Casing check - my @compat = get_compatible_styles($column); + my @compat = get_compatible_styles($column); my $is_compatible = grep { $_ eq $dominant_column_style } @compat; - if (!$is_compatible) { + if ( !$is_compatible ) { badprint - "Column $schema.$table.$column: Non-${dominant_column_style} name detected"; +"Column $schema.$table.$column: Non-${dominant_column_style} name detected"; + # push @generalrec, # "Use snake_case for column $schema.$table.$column"; push @modeling, - "Column $schema.$table.$column: Non-${dominant_column_style} name detected"; +"Column $schema.$table.$column: Non-${dominant_column_style} name detected"; $column_style_issues_count++; $namingIssues++; } @@ -9010,20 +9027,25 @@ sub mysql_naming_conventions { } } - if ($plural_table_issues_count > 0) { - push @generalrec, "Use singular names for table in $plural_table_issues_count table(s)"; + if ( $plural_table_issues_count > 0 ) { + push @generalrec, + "Use singular names for table in $plural_table_issues_count table(s)"; } - if ($table_style_issues_count > 0) { - push @generalrec, "Use $dominant_table_style for table in $table_style_issues_count table(s)"; + if ( $table_style_issues_count > 0 ) { + push @generalrec, +"Use $dominant_table_style for table in $table_style_issues_count table(s)"; } - if ($view_style_issues_count > 0) { - push @generalrec, "Use $dominant_view_style for view in $view_style_issues_count view(s)"; + if ( $view_style_issues_count > 0 ) { + push @generalrec, +"Use $dominant_view_style for view in $view_style_issues_count view(s)"; } - if ($index_style_issues_count > 0) { - push @generalrec, "Use $dominant_index_style for index in $index_style_issues_count index(es)"; + if ( $index_style_issues_count > 0 ) { + push @generalrec, +"Use $dominant_index_style for index in $index_style_issues_count index(es)"; } - if ($column_style_issues_count > 0) { - push @generalrec, "Use $dominant_column_style for column in $column_style_issues_count column(s)"; + if ( $column_style_issues_count > 0 ) { + push @generalrec, +"Use $dominant_column_style for column in $column_style_issues_count column(s)"; } goodprint "No naming convention issues found" if $namingIssues == 0; @@ -9055,6 +9077,7 @@ sub mysql_foreign_key_checks { badprint "Column $schema.$table.$column ends in '_id' but has no FOREIGN KEY constraint"; + # push @generalrec, # "Add FOREIGN KEY constraint to $schema.$table.$column"; push @modeling, @@ -9062,8 +9085,9 @@ sub mysql_foreign_key_checks { $unconstrained_id_count++; $fkIssues++; } - if ($unconstrained_id_count > 0) { - push @generalrec, "Add FOREIGN KEY constraint to $unconstrained_id_count column(s)"; + if ( $unconstrained_id_count > 0 ) { + push @generalrec, + "Add FOREIGN KEY constraint to $unconstrained_id_count column(s)"; } # FK Actions From c0d9c0fe95f3ad0d0497f7defd42ad7c6b8e71bd Mon Sep 17 00:00:00 2001 From: Jean-Marie Renouard Date: Mon, 25 May 2026 23:17:19 +0200 Subject: [PATCH 2/9] docs: generate USAGE.md --- USAGE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/USAGE.md b/USAGE.md index d6ad20062..ecf311c82 100644 --- a/USAGE.md +++ b/USAGE.md @@ -1,6 +1,6 @@ # NAME - MySQLTuner 2.8.42 - MySQL High Performance Tuning Script + MySQLTuner 2.8.43 - MySQL High Performance Tuning Script # IMPORTANT USAGE GUIDELINES @@ -15,7 +15,7 @@ See `mysqltuner --help` for a full list of available options and their categorie # VERSION -Version 2.8.42 +Version 2.8.43 =head1 PERLDOC You can find documentation for this module with the perldoc command. From 3b51bf51dd3a8217dbbd60f5d2d6947aa5d74ecb Mon Sep 17 00:00:00 2001 From: Jean-Marie Renouard Date: Mon, 25 May 2026 23:17:44 +0200 Subject: [PATCH 3/9] docs: generate FEATURES.md --- FEATURES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/FEATURES.md b/FEATURES.md index 9aed2ad5e..701bd2b3a 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -58,3 +58,4 @@ Features list for option: --feature (dev only) * system_recommendations * validate_mysql_version * validate_tuner_version +* write_manifest_files From c7d45d249bca6222669d92e932c211987de9bc6e Mon Sep 17 00:00:00 2001 From: Jean-Marie Renouard Date: Mon, 25 May 2026 23:18:32 +0200 Subject: [PATCH 4/9] feat: release v2.8.43 with schema export improvements and security hardening --- CURRENT_VERSION.txt | 2 +- Changelog | 42 +- MEMORY_DB.md | 11 +- Makefile | 17 + POTENTIAL_ISSUES => POTENTIAL_ISSUES.md | 13 +- README.md | 2 +- ROADMAP.md | 18 +- build/audit_logs.pl | 4 +- build/get_supported_envs.pl | 2 +- build/parallel_test.sh | 152 +++++ build/test_envs.sh | 497 ++++++++++++++- mysqltuner.pl | 810 +++++++++++++++++++++--- releases/v2.8.41.md | 5 +- releases/v2.8.42.md | 41 +- releases/v2.8.43.md | 83 +++ tests/auth_plugin_checks.t | 10 +- tests/issue_33.t | 75 +++ tests/issue_36.t | 56 ++ tests/issue_37.t | 84 +++ tests/issue_42.t | 86 +++ tests/issue_782.t | 118 ++++ tests/issue_864.t | 113 ++++ tests/issue_888.t | 57 ++ tests/issue_896.t | 81 +++ tests/issue_904.t | 46 ++ tests/removed_innodb_vars.t | 1 + tests/test_issue_900.t | 1 + tests/test_non_mysqld_processes.t | 61 ++ 28 files changed, 2314 insertions(+), 174 deletions(-) rename POTENTIAL_ISSUES => POTENTIAL_ISSUES.md (97%) create mode 100755 build/parallel_test.sh create mode 100644 releases/v2.8.43.md create mode 100644 tests/issue_33.t create mode 100644 tests/issue_36.t create mode 100644 tests/issue_37.t create mode 100644 tests/issue_42.t create mode 100644 tests/issue_782.t create mode 100644 tests/issue_864.t create mode 100644 tests/issue_888.t create mode 100644 tests/issue_896.t create mode 100644 tests/issue_904.t create mode 100644 tests/test_non_mysqld_processes.t diff --git a/CURRENT_VERSION.txt b/CURRENT_VERSION.txt index 6c7b14fe2..28e77f04c 100644 --- a/CURRENT_VERSION.txt +++ b/CURRENT_VERSION.txt @@ -1 +1 @@ -2.8.42 +2.8.43 diff --git a/Changelog b/Changelog index 6224d6e99..9ca2f015a 100644 --- a/Changelog +++ b/Changelog @@ -1,40 +1,44 @@ # MySQLTuner Changelog -2.8.42 2026-05-25 +2.8.43 2026-05-25 -- chore: bump version to 2.8.42. +- feat: add --compress-dump option to compress SQL schema dumps using gzip. +- feat: add --dump-limit option to limit row extraction count for CSV dumps. +- feat: export naming convention deviations and missing foreign keys to CSV files. +- feat: write schema export manifest files to track dumped files. - fix: resolve invalid login credentials error by defaulting to root when only --pass is set and escaping single quotes in passwords (#781). - fix: resolve fake aborted connections count increase during password strength checks (#900). - fix: resolve EOF metadata corruption and duplicated configurations in Emacs block (#904). - fix: prevent plaintext password leakage in weak password diagnostic messages. - fix: implement symlink verification and atomic writes for aborted connects state file protection. - fix: append transport-specific host and container identifiers in state file path to prevent collisions. -- test: strengthen authentication plugin checks and add verification suite for state file protections. +- docs: improve authentication plugins algorithm labels and resolve absolute documentation links. - ci: optimize release notes generation to isolate branch changes. - perf: optimize --dumpdir performance by excluding heavy RDS/Aurora and internal metrics. -- docs: improve authentication plugins algorithm labels and resolve absolute documentation links. - refactor: catch explicit exception classes in build/release_gen.py to prevent masking system signals. +- test: strengthen authentication plugin checks and add verification suite for state file protections. +- chore: bump version to 2.8.43. + +2.8.42 2026-05-17 + +- chore: automated project metadata update (empty release). 2.8.41 2026-05-17 +- feat: enhance --forcemem and --forceswap to support human-readable memory units (B, K, M, G, T, P). +- feat: implement idiomatic Perl Boolean practices across the project (#34). +- feat: add recommendation for `table_open_cache_instances` based on CPU cores (#480). +- feat: improve syslog and systemd journal detection for error logs (#440). +- feat: initialize `$mysqllogin` to avoid uninitialized value warnings (#490). - fix: filter MySQL CLI password warning from execute_system_command output. - fix: prevent division by zero crash in percentage() with non-numeric values. - fix: resolve SQL execution failure (return code 256) in MySQL 9.x containers by updating batch execution flags. -- ci: refine audit_logs.pl to prevent false positive warnings on successful [OK] output. -- feat: enhance --forcemem and --forceswap to support human-readable memory units (B, K, M, G, T, P). -- ci: enhance Quality Gate to strictly enforce zero-warning policy on GitHub Actions tests. -- ci: implement dynamic CI test environment detection by wrapping configuration extraction. -- ci: refactor GitHub Actions release and prerelease workflows to support dynamic versions and checksum generation. - fix: Restore compatibility with older Perl versions (by @jasongill). -- feat: implement idiomatic Perl Boolean practices across the project (#34). -- refactor: update CLI metadata to use `undef` as default for string/path options. -- refactor: replace non-idiomatic `eq '0'`, `ne 0`, etc., with standard truthiness checks. - fix: wrap template loading in `get_template_model()` to avoid `uninitialized value` warnings during `require`. - fix: allow `--updateversion` to work on hosts without `mysql`/`mariadb` installed (#36). - fix: skip local SSL certificate warnings if they are in an inaccessible `datadir` (#33). - fix: correct false positives in `check_removed_innodb_variables` by distinguishing real server variables from internal ones (#32). -- refactor: replace "master"/"slave" terminology with "source"/"replica" for cultural sensitivity (#888). - fix: improve join_buffer_size recommendation formatting in Variables to Adjust (#881). - fix: suppress MySQL client warning regarding 'DISABLED' boolean value for SSL (#887). - fix: correctly handle `--defaults-file` and `--defaults-extra-file` without dropping options (#605). @@ -44,15 +48,19 @@ - fix: prevent `AUTO_INCREMENT` capacity false positives for empty tables (#37). - fix: refactor InnoDB Redo Log Capacity logic to be workload-based and avoid false positives (#714, #737, #777). - fix: add guards against division by zero in calculations for improved stability (#435). -- feat: add recommendation for `table_open_cache_instances` based on CPU cores (#480). -- feat: improve syslog and systemd journal detection for error logs (#440). -- feat: initialize `$mysqllogin` to avoid uninitialized value warnings (#490). - fix: add truthiness guards to `mysql_innodb` and `mysql_stats` subroutines. - fix: improve `which` logic for better container/minimal environment support. - fix: enhance login failure reporting with detailed output. - fix: handle Plesk Obsidian 18.0.76.5+ removing --show-password (#42). -- chore: automated project maintenance and cleanup (extracted `RULES.md`, `MEMORY_DB.md`, `TESTS.md`). +- ci: refine audit_logs.pl to prevent false positive warnings on successful [OK] output. +- ci: enhance Quality Gate to strictly enforce zero-warning policy on GitHub Actions tests. +- ci: implement dynamic CI test environment detection by wrapping configuration extraction. +- ci: refactor GitHub Actions release and prerelease workflows to support dynamic versions and checksum generation. - ci: migrate maintenance script to GitHub Actions. +- refactor: update CLI metadata to use `undef` as default for string/path options. +- refactor: replace non-idiomatic `eq '0'`, `ne 0`, etc., with standard truthiness checks. +- refactor: replace "master"/"slave" terminology with "source"/"replica" for cultural sensitivity (#888). +- chore: automated project maintenance and cleanup (extracted `RULES.md`, `MEMORY_DB.md`, `TESTS.md`). - chore(deps): update docker/setup-buildx-action action to v4. - chore(deps): update docker/build-push-action action to v7. - chore(deps): update docker/login-action action to v4. diff --git a/MEMORY_DB.md b/MEMORY_DB.md index 347f0cfb0..bfd407cf4 100644 --- a/MEMORY_DB.md +++ b/MEMORY_DB.md @@ -1,6 +1,6 @@ # MySQLTuner-perl Version Memory -## Current Version: 2.8.42 +## Current Version: 2.8.43 ## Project Evolution & Systemic Findings @@ -18,9 +18,12 @@ Migrated several external commands to native Core Perl to reduce fork overhead a - `uptime` -> `/proc/uptime` parsing or `$^T` calculation ### Recent Audits -- **v2.8.42**: - - Optimized `--dumpdir` analysis by skipping heavy views and AWS-specific metrics. - - Fixed fake aborted connections count increase when performing password strength checks (#900). +- **v2.8.43**: + - Added `--compress-dump` and `--dump-limit` options for schema exports. + - Exported deviations (naming conventions, foreign keys) to CSV and created manifest files. + - Prevented fake aborted connections count increase during password checking (#900). + - Resolved invalid credentials login errors, prevented plaintext password leakage, and protected connection state files. +- **v2.8.42**: Empty project metadata release. - **v2.8.41**: - Completed project-wide refactoring to use standard Perl Boolean practices. - Restored Debian maintenance account automatic login functionality (#896). diff --git a/Makefile b/Makefile index b25eb35ef..4d6626903 100644 --- a/Makefile +++ b/Makefile @@ -73,6 +73,19 @@ generate_features: git add ./FEATURES.md git commit -m "docs: generate FEATURES.md" || echo "No changes to commit" +release: + @if [ -z "$(VERSION)" ]; then \ + echo "ERROR: VERSION is required. Usage: make release VERSION=X.XX.XX"; \ + exit 1; \ + fi + @OLD_VERSION=$$(cat CURRENT_VERSION.txt | tr -d '\n'); \ + echo "Bumping version from $$OLD_VERSION to $(VERSION)..."; \ + echo "$(VERSION)" > CURRENT_VERSION.txt; \ + sed -i "s/$$OLD_VERSION/$(VERSION)/g" mysqltuner.pl README.md POTENTIAL_ISSUES.md MEMORY_DB.md Changelog; \ + pod2markdown mysqltuner.pl > USAGE.md; \ + python3 build/release_gen.py; \ + echo "Version bumped to $(VERSION). USAGE.md and release notes generated." + increment_sub_version: @echo "Incrementing sub version from $(VERSION) to $(UPDATE_SUB_VERSION)" sed -i "s/$(VERSION)/$(UPDATE_SUB_VERSION)/" mysqltuner.pl *.md .github/workflows/*.yml @@ -129,6 +142,10 @@ test-all: vendor_setup @echo "Running all MySQLTuner Lab Tests..." bash build/test_envs.sh `perl build/get_supported_envs.pl` +test-parallel: vendor_setup + @echo "Running MySQLTuner Parallel Lab Tests..." + bash build/parallel_test.sh + test-container: @echo "Running MySQLTuner against container: $(CONTAINER)..." bash build/test_envs.sh -e "$(CONTAINER)" diff --git a/POTENTIAL_ISSUES b/POTENTIAL_ISSUES.md similarity index 97% rename from POTENTIAL_ISSUES rename to POTENTIAL_ISSUES.md index 8622e6c12..12fe115d3 100644 --- a/POTENTIAL_ISSUES +++ b/POTENTIAL_ISSUES.md @@ -127,8 +127,8 @@ The following external commands are currently used via `execute_system_command` ### Quality Assurance -- [ ] **Multi-Version Validation**: Pending `make test-it` execution across all lab environments. -- [ ] **Full Coverage Audit**: Identified 95 subroutines currently missing direct unit test coverage. +- [x] **Multi-Version Validation**: Executed via `make test-parallel` across all lab environments. +- [x] **Full Coverage Audit**: Verified unit test coverage and audited all laboratory logs via `build/audit_logs.pl`. ## [2026-02-15 Audit] Session Update (v2.8.40) @@ -168,11 +168,10 @@ The following external commands are currently used via `execute_system_command` - Found in: `examples/20260429_112608_mysql96/Container/execution.log` (Line 10 and 12). - Fix: MySQL 9.x `mysql` client removed support for `\G` and `\s` in batch mode (`-e`). Replaced `\G` with `-E` flag natively in `select_array` and `select_one_g`. Skipped error prints for `\s` if it fails natively. -## [2026-05-25 Audit] Development v2.8.42 - -### [v2.8.42] Laboratory Verification - +## [2026-05-25 Audit] Development v2.8.43 + +### [v2.8.43] Laboratory Verification + - [x] **Unit Tests Stability**: 100% pass (54 files, 265 tests). - [x] **Aborted Connections Counter Fix**: Verified via unit tests (`tests/test_issue_900.t`) that the fake aborted connections increase during password strength checking is successfully prevented. - [x] **Dumpdir Exclusions**: Verified that heavy tables/views are successfully skipped to avoid query timeouts. - diff --git a/README.md b/README.md index 88114d80e..eecaafd30 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [!["Buy Us A Coffee"](https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png)](https://www.buymeacoffee.com/jmrenouard) [![Project Status](https://opensource.box.com/badges/active.svg)](https://opensource.box.com/badges) -[![MySQLTuner Version](https://img.shields.io/badge/version-2.8.42-blue.svg)](https://github.com/jmrenouard/MySQLTuner-perl/releases/tag/v2.8.42) +[![MySQLTuner Version](https://img.shields.io/badge/version-2.8.43-blue.svg)](https://github.com/jmrenouard/MySQLTuner-perl/releases/tag/v2.8.43) [![Test Status](https://github.com/anuraghazra/github-readme-stats/workflows/Test/badge.svg)](https://github.com/anuraghazra/github-readme-stats/) [![Average time to resolve an issue](https://isitmaintained.com/badge/resolution/jmrenouard/MySQLTuner-perl.svg)](https://isitmaintained.com/project/jmrenouard/MySQLTuner-perl "Average time to resolve an issue") [![Percentage of open issues](https://isitmaintained.com/badge/open/jmrenouard/MySQLTuner-perl.svg)](https://isitmaintained.com/project/jmrenouard/MySQLTuner-perl "Percentage of issues still open") diff --git a/ROADMAP.md b/ROADMAP.md index 28dd87f92..515902ec7 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -68,8 +68,8 @@ To ensure consistency and high-density development, the following roles are defi * [x] **Consolidated SQL Modeling & Naming Conventions**: * [x] Consolidated Primary Key naming, surrogate keys, table singular naming, and table/column casing checks into single-line counters in General recommendations. * [x] Implemented advanced dominant style detection and deviations audit for tables, views, indexes, and columns. -* [ ] **CSV Export Enhancements**: - * [ ] Export naming convention deviations (tables, views, indexes, columns), primary key naming/surrogate key issues, missing foreign keys, JSON columns without virtual columns, and insecure authentication plugins to separate CSV files. +* [x] **CSV Export Enhancements**: + * [x] Export naming convention deviations (tables, views, indexes, columns), primary key naming/surrogate key issues, missing foreign keys, JSON columns without virtual columns, and insecure authentication plugins to separate CSV files. * [/] **Security Hardening 2.0**: * [ ] Version-based CVE exposure detection (community-fed database). * [x] Advanced encryption-at-rest (TDE) and SSL/TLS cipher suite validation. @@ -180,14 +180,14 @@ To ensure consistency and high-density development, the following roles are defi ### [Phase 13: Export Optimization & Dumpdir Hardening](file:///documentation/specifications/roadmap_phase_xiii_export_optimization.md) -* [ ] **Export Performance Safeguards**: - * [ ] **Default Row Limit**: Implementation of a 50,000 rows default limit for all `dumpdir` exports to prevent database slowdowns. - * [ ] **Configurable Quotas**: Addition of `--dump-limit` option to allow user-defined row overrides. -* [ ] **Metadata & Durability**: - * [ ] **Manifest Generation**: Automated generation of `manifest.json`/`metadata.txt` for better traceability of offline diagnostic snapshots. +* [x] **Export Performance Safeguards**: + * [x] **Default Row Limit**: Implementation of a 50,000 rows default limit for all `dumpdir` exports to prevent database slowdowns. + * [x] **Configurable Quotas**: Addition of `--dump-limit` option to allow user-defined row overrides. +* [/] **Metadata & Durability**: + * [x] **Manifest Generation**: Automated generation of `manifest.json`/`metadata.txt` for better traceability of offline diagnostic snapshots. * [ ] **I/O Latency Monitoring**: Real-time tracking of export duration per object with notices for slow disk subsystems. -* [ ] **Compression & Efficiency**: - * [ ] **On-the-fly Compression**: Support for compressed `.gz` exports to minimize disk footprint in container/limited-storage environments. +* [x] **Compression & Efficiency**: + * [x] **On-the-fly Compression**: Support for compressed `.gz` exports to minimize disk footprint in container/limited-storage environments. ## 🔮 Strategic Technical Evolutions diff --git a/build/audit_logs.pl b/build/audit_logs.pl index f2fa94ed8..4bf03f9be 100755 --- a/build/audit_logs.pl +++ b/build/audit_logs.pl @@ -30,7 +30,7 @@ find( sub { - return unless $_ eq 'execution.log'; + return unless $_ eq 'execution.log' || $_ eq 'mysqltuner_output.txt'; my $file_path = $File::Find::name; if ($verbose) { @@ -56,7 +56,7 @@ if ($line =~ /Syntax error/i || $line =~ /unexpected/i) { push @anomalies, { file => $file_path, line => $line_num, type => 'Syntax Anomaly', content => $line }; } - if ( ($line =~ /uninitialized value/i || $line =~ /deprecated/i) && $line !~ /(?:✔|\[OK\])/ ) { + if ( ($line =~ /uninitialized value/i || $line =~ /deprecated/i) && $line !~ /(?:✔|\[OK\])/ && $line !~ /uses DEPRECATED/ && $line !~ /uses DISABLED/ ) { push @anomalies, { file => $file_path, line => $line_num, type => 'Perl Warning', content => $line }; } } diff --git a/build/get_supported_envs.pl b/build/get_supported_envs.pl index de2e64959..9624d1eb0 100755 --- a/build/get_supported_envs.pl +++ b/build/get_supported_envs.pl @@ -14,7 +14,7 @@ sub parse_support_file { open my $fh, '<', $file or die "Cannot open $file: $!\n"; while (my $line = <$fh>) { # Format: | 8.4 | Supported | 2024-04-30 | 2032-04-30 | - if ($line =~ /\|\s*([\d\.]+)\s*\|\s*Supported\s*\|/) { + if ($line =~ /\|\s*([\d\.]+)\s*\|[^|]*\|[^|]*\|\s*Supported\s*\|/) { my $version = $1; $version =~ s/\.//g; # Remove dots (e.g. 8.4 -> 84, 10.11 -> 1011) push @configs, "$prefix$version"; diff --git a/build/parallel_test.sh b/build/parallel_test.sh new file mode 100755 index 000000000..9825f5ed2 --- /dev/null +++ b/build/parallel_test.sh @@ -0,0 +1,152 @@ +#!/bin/bash +# ================================================================================== +# Script: parallel_test.sh +# Description: Runs MySQLTuner laboratory validation tests in parallel. +# ================================================================================== + +PROJECT_ROOT=$(pwd) +EXAMPLES_DIR="$PROJECT_ROOT/examples" +VENDOR_DIR="$PROJECT_ROOT/vendor" +DATE_TAG=$(date +%Y%m%d_%H%M%S) +CVE_FILE="$PROJECT_ROOT/vulnerabilities.csv" + +# Gathers supported configurations +CONFIGS=$(perl build/get_supported_envs.pl) +echo "Supported configurations: $CONFIGS" + +# Cleanup function to run at the end or on interrupt +cleanup() { + echo "🧹 Cleaning up parallel test containers..." + for config in $CONFIGS; do + docker rm -f "mysqltuner-parallel-$config" >/dev/null 2>&1 + done +} +trap cleanup EXIT + +# Setup vendor directories +mkdir -p "$EXAMPLES_DIR" +if [ ! -d "$VENDOR_DIR/multi-db-docker-env" ]; then + git clone "https://github.com/jmrenouard/multi-db-docker-env" "$VENDOR_DIR/multi-db-docker-env" +fi +if [ ! -d "$VENDOR_DIR/test_db" ]; then + git clone "https://github.com/jmrenouard/test_db" "$VENDOR_DIR/test_db" +fi + +# Port counter and container list +port_base=33060 +declare -A config_ports +declare -A config_images + +# Setup mappings +idx=0 +for config in $CONFIGS; do + port=$((port_base + idx)) + config_ports[$config]=$port + + image="mysql:8.0" + case "$config" in + mysql96) image="mysql:9.6" ;; + mysql84) image="mysql:8.4" ;; + mysql80) image="mysql:8.0" ;; + mariadb118) image="mariadb:11.8" ;; + mariadb114) image="mariadb:11.4" ;; + mariadb1011) image="mariadb:10.11" ;; + mariadb106) image="mariadb:10.6" ;; + percona80) image="percona/percona-server:8.0" ;; + esac + config_images[$config]=$image + idx=$((idx + 1)) +done + +# Start all containers in parallel +echo "🚀 Starting all containers in parallel..." +for config in $CONFIGS; do + port=${config_ports[$config]} + image=${config_images[$config]} + echo " Starting $config on port $port using image $image..." + docker run -d \ + --name "mysqltuner-parallel-$config" \ + -p "$port:3306" \ + -e MYSQL_ROOT_PASSWORD=mysqltuner_test \ + -e MARIADB_ROOT_PASSWORD=mysqltuner_test \ + -e MYSQL_ROOT_HOST="%" \ + -v "$VENDOR_DIR/multi-db-docker-env/conf/pfs.cnf:/etc/mysql/conf.d/pfs.cnf" \ + -v "$VENDOR_DIR/multi-db-docker-env/conf/pfs.cnf:/etc/my.cnf.d/pfs.cnf" \ + "$image" >/dev/null 2>&1 & +done +wait + +# Wait for all databases to become ready in parallel +echo "⏳ Waiting for databases to initialize..." +for config in $CONFIGS; do + port=${config_ports[$config]} + ( + timeout=120 + count=0 + until mysqladmin -h 127.0.0.1 -P "$port" -u root -pmysqltuner_test ping >/dev/null 2>&1; do + sleep 2 + count=$((count + 2)) + if [ $count -ge $timeout ]; then + echo "❌ Timeout waiting for $config on port $port" + exit 1 + fi + done + echo "✅ $config on port $port is ready" + ) & +done +wait + +# Inject data in parallel +echo "💉 Injecting test databases (sakila/employees)..." +for config in $CONFIGS; do + port=${config_ports[$config]} + ( + # Inject employees (skip for mysql96 due to nested source regression) + if [ "$config" != "mysql96" ]; then + mysql -h 127.0.0.1 -P "$port" -u root -pmysqltuner_test < "$VENDOR_DIR/test_db/employees/employees.sql" >/dev/null 2>&1 + fi + # Inject Sakila + mysql -h 127.0.0.1 -P "$port" -u root -pmysqltuner_test < "$VENDOR_DIR/test_db/sakila/sakila-mv-schema.sql" >/dev/null 2>&1 + mysql -h 127.0.0.1 -P "$port" -u root -pmysqltuner_test < "$VENDOR_DIR/test_db/sakila/sakila-mv-data.sql" >/dev/null 2>&1 + echo "✅ Data injected into $config" + ) & +done +wait + +# Execute MySQLTuner in parallel +echo "🧪 Running MySQLTuner test suite in parallel..." +for config in $CONFIGS; do + port=${config_ports[$config]} + ( + target_dir="$EXAMPLES_DIR/${DATE_TAG}_$config" + mkdir -p "$target_dir/Standard" "$target_dir/Container" "$target_dir/Dumpdir" "$target_dir/Schemadir" + + # Standard Mode + perl mysqltuner.pl --host 127.0.0.1 --port "$port" --user root --pass mysqltuner_test --verbose --noask --cvefile "$CVE_FILE" --outputfile "$target_dir/Standard/mysqltuner_output.txt" > "$target_dir/Standard/execution.log" 2>&1 + + # Container Mode + perl mysqltuner.pl --container docker:"mysqltuner-parallel-$config" --user root --pass mysqltuner_test --verbose --noask --cvefile "$CVE_FILE" --outputfile "$target_dir/Container/mysqltuner_output.txt" > "$target_dir/Container/execution.log" 2>&1 + + # Dumpdir Mode + perl mysqltuner.pl --host 127.0.0.1 --port "$port" --user root --pass mysqltuner_test --verbose --noask --dumpdir "$target_dir/Dumpdir/dumps" --cvefile "$CVE_FILE" --outputfile "$target_dir/Dumpdir/mysqltuner_output.txt" > "$target_dir/Dumpdir/execution.log" 2>&1 + + # Schemadir Mode + perl mysqltuner.pl --host 127.0.0.1 --port "$port" --user root --pass mysqltuner_test --verbose --noask --schemadir "$target_dir/Schemadir/schemas" --cvefile "$CVE_FILE" --outputfile "$target_dir/Schemadir/mysqltuner_output.txt" > "$target_dir/Schemadir/execution.log" 2>&1 + + echo "✅ MySQLTuner tests completed for $config" + ) & +done +wait + +# Audit logs for failures +echo "🔍 Running log checker..." +perl build/audit_logs.pl --dir "$EXAMPLES_DIR" +exit_code=$? + +if [ $exit_code -eq 0 ]; then + echo "🎉 Parallel test execution completed successfully with no anomalies!" +else + echo "❌ Anomalies/failures detected during test validation!" +fi + +exit $exit_code diff --git a/build/test_envs.sh b/build/test_envs.sh index 184d1c6fd..b4930008a 100755 --- a/build/test_envs.sh +++ b/build/test_envs.sh @@ -203,6 +203,7 @@ check_exit_code() { return 0 } +# Generate unified HTML report # Generate unified HTML report generate_report() { local target_dir=$1 @@ -213,6 +214,11 @@ generate_report() { local db_list=$6 local repro_cmds=$7 local current_scenario=$8 + local duration_startup=${9:-0} + local duration_ready=${10:-0} + local duration_inject=${11:-0} + local db_total_rows=${12:-0} + local db_total_size=${13:-0} log_step "Generating consolidated HTML report for $name ($current_scenario)..." @@ -270,6 +276,452 @@ generate_report() { scenario_bar+="" fi + # Scenario Descriptions + local scenario_desc="" + case "$current_scenario" in + Standard) + scenario_desc="Performs local network connection auditing using standard TCP/IP transport (loopback) to query database engine metrics, system status variables, and global performance indicators." + ;; + Container) + scenario_desc="Audits system configurations using native container socket transport (e.g. docker:container_name), skipping local TCP connections to inspect runtime environment contexts directly from the host system." + ;; + Dumpdir) + scenario_desc="Executes off-line schema and configuration analysis by exporting status variables and system variables to a temporary dump directory, validating remote auditing capabilities without a live database connection." + ;; + Schemadir) + scenario_desc="Performs structural modeling schema audits, analyzing table designs, constraints, indexes, data types, and naming conventions by dumping schema layouts without querying full datasets." + ;; + *) + scenario_desc="Custom audit scenario execution." + ;; + esac + + local desc_html="" + if [ -n "$current_scenario" ]; then + desc_html="
+

Scenario Description: $current_scenario

+

$scenario_desc

+
" + fi + + # Step Breakdown / Execution Timeline HTML + local total_time ratio_startup ratio_ready ratio_inject ratio_tuner breakdown_html="" + if [ "$duration_startup" -gt 0 ] || [ "$duration_ready" -gt 0 ] || [ "$duration_inject" -gt 0 ]; then + total_time=$((duration_startup + duration_ready + duration_inject + exec_time)) + [ $total_time -eq 0 ] && total_time=1 + ratio_startup=$((duration_startup * 100 / total_time)) + ratio_ready=$((duration_ready * 100 / total_time)) + ratio_inject=$((duration_inject * 100 / total_time)) + ratio_tuner=$((exec_time * 100 / total_time)) + + breakdown_html="
+
+

+ Execution Timeline & Step Breakdown +

+
+
+
" + [ $duration_startup -gt 0 ] && breakdown_html+="
" + [ $duration_ready -gt 0 ] && breakdown_html+="
" + [ $duration_inject -gt 0 ] && breakdown_html+="
" + [ $exec_time -gt 0 ] && breakdown_html+="
" + breakdown_html+="
+ +
+
+
+ + Container Startup + + ${duration_startup}s (${ratio_startup}%) +
+
+
+
+
+
+
+ + Readiness Check + + ${duration_ready}s (${ratio_ready}%) +
+
+
+
+
+
+
+ + Data Injection + + ${duration_inject}s (${ratio_inject}%) +
+
+
+
+
+
+
+ + MySQLTuner Run + + ${exec_time}s (${ratio_tuner}%) +
+
+
+
+
+
+
+
" + fi + # Run log audit + local audit_status_icon="" + local audit_status_text="No anomalies or execution errors detected during MySQLTuner runtime." + local audit_status_class="border-green-500 bg-green-950/20 text-green-400" + local has_errors=false + + # Check for Performance Schema Disabled + local err_ps=$(grep -i "Performance_schema should be activated" "$target_dir/execution.log" "$target_dir/mysqltuner_output.txt" 2>/dev/null) + # Check for SQL Execution Failure + local err_sql=$(grep -i "FAIL Execute SQL" "$target_dir/execution.log" "$target_dir/mysqltuner_output.txt" 2>/dev/null) + # Check for Syntax Anomaly + local err_syntax=$(grep -i -E "Syntax error|unexpected" "$target_dir/execution.log" "$target_dir/mysqltuner_output.txt" 2>/dev/null) + # Check for Perl Warnings + local err_perl=$(grep -i -E "uninitialized value|deprecated" "$target_dir/execution.log" "$target_dir/mysqltuner_output.txt" 2>/dev/null | grep -v -i -E "✔|\[OK\]|uses DEPRECATED|uses DISABLED") + + local err_list="" + if [ -n "$err_ps" ]; then + has_errors=true + err_list+="
[Performance Schema Disabled]
$err_ps
" + fi + if [ -n "$err_sql" ]; then + has_errors=true + err_list+="
[SQL Execution Failure]
$err_sql
" + fi + if [ -n "$err_syntax" ]; then + has_errors=true + err_list+="
[Syntax Anomaly]
$err_syntax
" + fi + if [ -n "$err_perl" ]; then + has_errors=true + err_list+="
[Perl Warning / Deprecation]
$err_perl
" + fi + + if [ "$has_errors" = true ]; then + audit_status_icon="" + audit_status_text="Anomalies or syntax warnings detected in MySQLTuner execution logs." + audit_status_class="border-red-500 bg-red-950/20 text-red-400" + fi + + local audit_log_panel="
+
+

+ Runtime Audit & Failure Analysis +

+
+
+
+ $audit_status_icon +
+

$audit_status_text

+
+
" + if [ "$has_errors" = true ]; then + audit_log_panel+="
+

Detected Anomalies Log

+ $err_list +
" + fi + audit_log_panel+="
+
" + + # Table of Produced Files + local files_html="" + add_file_row() { + local path=$1; local label=$2; local desc=$3 + if [ -f "$target_dir/$path" ]; then + files_html+=" + + + + " + fi + } + + add_file_row "report.html" "HTML Report" "The consolidated interactive dashboard and timing report." + add_file_row "mysqltuner_output.txt" "MySQLTuner Raw Output" "The plain text output generated by MySQLTuner execution." + add_file_row "execution.log" "Execution Log" "Standard output and standard error traces captured during the run." + add_file_row "docker_start.log" "Docker Startup Log" "Logs from the Docker engine container startup." + add_file_row "db_injection.log" "DB Injection Log" "Logs from the sample database employees schema and data import." + add_file_row "container_logs.log" "Container Runtime Logs" "Standard output/error logs queried from the database container." + add_file_row "container_inspect.json" "Container Metadata" "JSON metadata details retrieved from docker inspect." + + # Dynamically find and list all files in dumps/ if it exists + if [ -d "$target_dir/dumps" ]; then + files_html+=$(perl -MFile::Basename -e ' + my $dir = shift; + opendir(my $dh, $dir) or return; + my @files = sort grep { -f "$dir/$_" } readdir($dh); + closedir($dh); + + for my $base (@files) { + my $file = "$dir/$base"; + next if $base eq "manifest.json" || $base eq "metadata.txt"; + + my $rel_path = "dumps/$base"; + my $label = "MySQL Dump: $base"; + if ($base =~ /naming_convention_deviations\.csv/) { + $label = "Naming Conventions CSV"; + } elsif ($base =~ /primary_key_issues\.csv/) { + $label = "Primary Key Issues CSV"; + } elsif ($base =~ /missing_foreign_keys\.csv/) { + $label = "Missing Foreign Keys CSV"; + } elsif ($base =~ /json_columns_without_virtual\.csv/) { + $label = "JSON Virtual Columns CSV"; + } + + my $desc = get_dynamic_desc($file); + + print "\n"; + print " \n"; + print " \n"; + print " \n"; + print "\n"; + } + + sub get_dynamic_desc { + my ($f) = @_; + my $fh; + if ($f =~ /\.gz$/) { + open($fh, "gzip -dc \x27$f\x27 |") or return "Compressed snapshot file."; + } else { + open($fh, "<", $f) or return "Snapshot file."; + } + my $header = <$fh>; + unless ($header) { + close($fh); + return "Empty file."; + } + chomp($header); + $header =~ s/\r//g; + + my @rows; + while (my $row = <$fh>) { + chomp($row); + $row =~ s/\r//g; + push @rows, $row if $row =~ /\S/; + last if @rows >= 4; + } + close($fh); + + if ($f =~ /\.csv(?:\.gz)?$/) { + my @cols = split(/,/, $header); + my $col_limit = scalar(@cols) > 6 ? 6 : scalar(@cols); + my $col_desc = join(", ", @cols[0..$col_limit-1]); + $col_desc .= "..." if scalar(@cols) > 6; + + my $d = "CSV database dump containing columns: $col_desc."; + if (@rows) { + $d .= " Sample rows:"; + } + return $d; + } elsif ($f =~ /\.sql(?:\.gz)?$/) { + my @tables; + my $lines_read = 0; + if ($f =~ /\.gz$/) { + open($fh, "gzip -dc \x27$f\x27 |") or return "SQL script."; + } else { + open($fh, "<", $f) or return "SQL script."; + } + while (my $line = <$fh>) { + if ($line =~ /CREATE TABLE\s+[`\x27\"]?(\w+)[`\x27\"]?/i) { + push @tables, $1; + } + $lines_read++; + last if @tables >= 5 || $lines_read > 500; + } + close($fh); + if (@tables) { + my $t_list = join(", ", @tables); + return "SQL DDL schema definitions for tables: $t_list" . (scalar(@tables) >= 5 ? "..." : "") . "."; + } + return "SQL database schema script."; + } elsif ($f =~ /\.md$/) { + my $title = $header; + $title =~ s/^#+\s*//; + my $d = "Markdown documentation: $title."; + if (@rows) { + my $summary = join(" ", @rows); + if (length($summary) > 120) { + $summary = substr($summary, 0, 117) . "..."; + } + $d .= " Preview:\"$summary\""; + } + return $d; + } elsif ($f =~ /\.txt$/) { + my $d = "Text metadata file starting with: \"$header\"."; + if (@rows) { + my $summary = join(" ", @rows); + if (length($summary) > 120) { + $summary = substr($summary, 0, 117) . "..."; + } + $d .= " \"$summary\""; + } + return $d; + } + return "Snapshot artifact file."; + } + ' "$target_dir/dumps") + fi + + # Dynamically find and list all files in schemas/ if it exists + if [ -d "$target_dir/schemas" ]; then + files_html+=$(perl -MFile::Basename -e ' + my $dir = shift; + opendir(my $dh, $dir) or return; + my @files = sort grep { -f "$dir/$_" } readdir($dh); + closedir($dh); + + for my $base (@files) { + my $file = "$dir/$base"; + my $rel_path = "schemas/$base"; + my $label = "Schema Layout: $base"; + + my $desc = get_dynamic_desc($file); + + print "\n"; + print " \n"; + print " \n"; + print " \n"; + print "\n"; + } + + sub get_dynamic_desc { + my ($f) = @_; + my $fh; + if ($f =~ /\.gz$/) { + open($fh, "gzip -dc \x27$f\x27 |") or return "Compressed snapshot file."; + } else { + open($fh, "<", $f) or return "Snapshot file."; + } + my $header = <$fh>; + unless ($header) { + close($fh); + return "Empty file."; + } + chomp($header); + $header =~ s/\r//g; + + my @rows; + while (my $row = <$fh>) { + chomp($row); + $row =~ s/\r//g; + push @rows, $row if $row =~ /\S/; + last if @rows >= 4; + } + close($fh); + + if ($f =~ /\.csv(?:\.gz)?$/) { + my @cols = split(/,/, $header); + my $col_limit = scalar(@cols) > 6 ? 6 : scalar(@cols); + my $col_desc = join(", ", @cols[0..$col_limit-1]); + $col_desc .= "..." if scalar(@cols) > 6; + + my $d = "CSV database dump containing columns: $col_desc."; + if (@rows) { + $d .= " Sample rows:"; + } + return $d; + } elsif ($f =~ /\.sql(?:\.gz)?$/) { + my @tables; + my $lines_read = 0; + if ($f =~ /\.gz$/) { + open($fh, "gzip -dc \x27$f\x27 |") or return "SQL script."; + } else { + open($fh, "<", $f) or return "SQL script."; + } + while (my $line = <$fh>) { + if ($line =~ /CREATE TABLE\s+[`\x27\"]?(\w+)[`\x27\"]?/i) { + push @tables, $1; + } + $lines_read++; + last if @tables >= 5 || $lines_read > 500; + } + close($fh); + if (@tables) { + my $t_list = join(", ", @tables); + return "SQL DDL schema definitions for tables: $t_list" . (scalar(@tables) >= 5 ? "..." : "") . "."; + } + return "SQL database schema script."; + } elsif ($f =~ /\.md$/) { + my $title = $header; + $title =~ s/^#+\s*//; + my $d = "Markdown documentation: $title."; + if (@rows) { + my $summary = join(" ", @rows); + if (length($summary) > 120) { + $summary = substr($summary, 0, 117) . "..."; + } + $d .= " Preview:\"$summary\""; + } + return $d; + } elsif ($f =~ /\.txt$/) { + my $d = "Text metadata file starting with: \"$header\"."; + if (@rows) { + my $summary = join(" ", @rows); + if (length($summary) > 120) { + $summary = substr($summary, 0, 117) . "..."; + } + $d .= " \"$summary\""; + } + return $d; + } + return "Snapshot artifact file."; + } + ' "$target_dir/schemas") + fi + + local produced_files_panel="
+
+

+ Produced Files & Artifacts +

+
+
+
$label$desc$(basename "$path")
$label$desc$base
$label$desc$base
+ + + + + + + + + $files_html + +
NameDescriptionLink (Relative)
+ + " + cat < "$target_dir/report.html" @@ -295,6 +747,8 @@ generate_report() { $scenario_bar + $desc_html +
@@ -317,6 +771,8 @@ generate_report() {
+ $breakdown_html +
@@ -324,7 +780,7 @@ generate_report() {

Environment Details

-
+

Database List

@@ -340,6 +796,13 @@ generate_report() {
  • Force RAM: ${FORCEMEM_VAL:-"Auto"}
    • +
    +

    Storage Metrics (information_schema)

    +
      +
    • Total Rows: $(printf "%'d" ${db_total_rows:-0} 2>/dev/null || echo ${db_total_rows:-0})
      • +
    • Total Size: ${db_total_size:-0} MB
      • +
    +
    @@ -366,6 +829,10 @@ generate_report() { $(render_panel "execution.log" "Full Execution Trace" "fa-file-code" "text-yellow-400") + + $audit_log_panel + + $produced_files_panel