diff --git a/Makefile b/Makefile index 7c148180..f42719c7 100644 --- a/Makefile +++ b/Makefile @@ -76,7 +76,7 @@ else ifeq ($(UNAME_S)-$(UNAME_M),Linux-aarch64) FORKPRESS_TARGET ?= aarch64-unknown-linux-musl endif -.PHONY: all clean test test-compat test-branchfs test-cow test-cow-branch-birth test-cow-branch-ui test-cow-changed test-cow-e2e-git-existing-update-crash test-cow-e2e-remote-cache test-cow-e2e-semantic test-cow-explicit-ids test-cow-fast test-cow-filesystem test-cow-git-server test-cow-id-bands test-cow-media-validator test-cow-merge test-cow-merge-smoke test-cow-plugin-validator test-cow-schema-review test-cow-semantic-fast test-cow-stale-audit test-cow-wp-semantic-validator test-branch-cli-fast test-release init-db test-all forkpress forkpress-dev dist dist-dev +.PHONY: all clean test test-compat test-branchfs test-cow test-cow-branch-birth test-cow-branch-ui test-cow-changed test-cow-e2e-git-existing-update-crash test-cow-e2e-remote-cache test-cow-e2e-semantic test-cow-explicit-ids test-cow-fast test-cow-filesystem test-cow-git-server test-cow-id-bands test-cow-media-validator test-cow-merge test-cow-merge-smoke test-cow-mysql-import test-cow-plugin-validator test-cow-schema-review test-cow-semantic-fast test-cow-stale-audit test-cow-wp-semantic-validator test-branch-cli-fast test-release init-db test-all forkpress forkpress-dev dist dist-dev all: $(BRANCHFS_EXT_SO) @@ -114,6 +114,9 @@ test-cow-changed: test-cow-merge: test-cow-merge-smoke php $(COW_TEST_DIR)/merge.php +test-cow-mysql-import: + php $(COW_TEST_DIR)/mysql_import.php + test-cow-git-server: php $(COW_TEST_DIR)/git_server.php @@ -170,7 +173,7 @@ test-cow-semantic-fast: test-cow-merge-smoke php $(COW_TEST_DIR)/stale_audit.php php $(COW_TEST_DIR)/wp_semantic_validator.php -test-cow-fast: test-cow-git-server test-cow-merge-smoke +test-cow-fast: test-cow-git-server test-cow-merge-smoke test-cow-mysql-import php $(COW_TEST_DIR)/branch_birth.php php $(COW_TEST_DIR)/explicit_ids.php php $(COW_TEST_DIR)/filesystem.php diff --git a/README.md b/README.md index ca17a1f1..77551189 100644 --- a/README.md +++ b/README.md @@ -229,8 +229,10 @@ forkpress remote clone production \ ``` The first sync is thin by default: uploads, caches, backups, logs, and upgrade -temp files are skipped so the branch can boot quickly. Use `--include-uploads` -or `--full-sync` when you need more of the remote tree locally. +temp files are skipped so the branch can boot quickly. ForkPress can branch from +an existing ForkPress SQLite sidecar or import a normal MySQL-backed WordPress +database over the same SSH connection. Use `--include-uploads` or `--full-sync` +when you need more of the remote tree locally. Install plugins from the branch's WordPress admin at `/wp-admin/plugin-install.php`. ForkPress tracks the WordPress.org top 100 @@ -480,7 +482,7 @@ Useful log files: | `forkpress agents [dir]` | Create agent branches and Git worktrees. | | `forkpress commit -m "message"` | Commit and push the current Git branch back to ForkPress. | | `forkpress pull` | Pull with rebase and autostash. | -| `forkpress remote clone --ssh --path --branch ` | Thin-clone a boot-ready remote WordPress root over SSH, skipping uploads, caches, backups, logs, and upgrade temp files by default, then create a local COW branch. | +| `forkpress remote clone --ssh --path --branch ` | Thin-clone a boot-ready remote WordPress root over SSH, import MySQL into SQLite when needed, then create a local COW branch. | | `forkpress logs --file ` | Read WordPress, PHP, server, and maintenance logs. | | `forkpress storage status` | Show selected storage and mount state. | | `forkpress storage mount` | Attach mount-backed storage. | diff --git a/crates/forkpress-cli/build.rs b/crates/forkpress-cli/build.rs index b70814cf..13afd0e1 100644 --- a/crates/forkpress-cli/build.rs +++ b/crates/forkpress-cli/build.rs @@ -71,6 +71,8 @@ fn main() -> Result<()> { "runtime/wp.zip", "scripts/cow/git_server.php", "scripts/cow/merge.php", + "scripts/cow/mysql_export.php", + "scripts/cow/mysql_import_sqlite.php", "scripts/git/autoload.php", "scripts/shared/sqlite_backup.php", "scripts/shared/sqlite_retry.php", @@ -466,6 +468,8 @@ fn build_bundle( } else { add_file(&mut tar, repo_root, "scripts/cow/git_server.php")?; add_file(&mut tar, repo_root, "scripts/cow/merge.php")?; + add_file(&mut tar, repo_root, "scripts/cow/mysql_export.php")?; + add_file(&mut tar, repo_root, "scripts/cow/mysql_import_sqlite.php")?; add_file(&mut tar, repo_root, "scripts/git/autoload.php")?; add_file(&mut tar, repo_root, "scripts/shared/sqlite_backup.php")?; add_file(&mut tar, repo_root, "scripts/shared/sqlite_retry.php")?; diff --git a/crates/forkpress-cli/src/app.rs b/crates/forkpress-cli/src/app.rs index afbbca44..bc1d37d4 100644 --- a/crates/forkpress-cli/src/app.rs +++ b/crates/forkpress-cli/src/app.rs @@ -29,7 +29,6 @@ use forkpress_git::{ }; #[cfg(feature = "dev-experiments")] use forkpress_runtime::run_php_script; -#[cfg(feature = "dev-experiments")] use forkpress_runtime::write_filtered_output; use forkpress_runtime::{ PortableRuntime, php_base_command, prepare_runtime as prepare_embedded_runtime, @@ -2847,6 +2846,16 @@ fn remote_clone_command( cache_root.display() ); } + if !cache_root.join("wp-content/database/.ht.sqlite").is_file() + && cache_root.join("wp-config.php").is_file() + { + prepare_runtime(layout)?; + let runtime = PortableRuntime::from_layout(layout); + remote_clone_import_mysql_database(shared, layout, &runtime, &args, &cache_root) + .with_context( + || "remote site did not include ForkPress SQLite data and MySQL import failed", + )?; + } let manifest = add_remote_site( layout, @@ -2925,6 +2934,90 @@ fn remote_clone_cache_root(layout: &Layout, name: &str) -> Result { Ok(layout.cow_dir.join("remote-sites").join(name).join("cache")) } +fn remote_clone_import_mysql_database( + shared: &SharedPaths, + layout: &Layout, + runtime: &PortableRuntime, + args: &RemoteCloneArgs, + cache_root: &Path, +) -> Result<()> { + let export_path = cache_root + .parent() + .ok_or_else(|| anyhow!("failed to resolve remote cache parent"))? + .join("mysql-export.jsonl"); + let export_file = File::create(&export_path) + .with_context(|| format!("failed to create {}", export_path.display()))?; + let exporter = fs::read_to_string(layout.runtime_dir.join("scripts/cow/mysql_export.php")) + .context("failed to read bundled MySQL export helper")?; + + let mut command = remote_clone_ssh_command(args); + command + .arg(&args.ssh) + .arg(format!("php -- {}", shell_quote(&args.remote_path))) + .stdin(Stdio::piped()) + .stdout(Stdio::from(export_file)) + .stderr(Stdio::piped()); + let mut child = command + .spawn() + .context("failed to start ssh for remote MySQL export")?; + if let Some(mut stdin) = child.stdin.take() { + stdin + .write_all(exporter.as_bytes()) + .context("failed to send MySQL export helper over ssh")?; + } + let output = child + .wait_with_output() + .context("failed to wait for remote MySQL export")?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + bail!( + "remote MySQL export failed with status {}{}{}", + output.status, + if stderr.trim().is_empty() { "" } else { ": " }, + stderr.trim() + ); + } + let metadata = fs::metadata(&export_path) + .with_context(|| format!("failed to stat {}", export_path.display()))?; + if metadata.len() == 0 { + bail!("remote MySQL export produced no data"); + } + + let db_path = cache_root.join("wp-content/database/.ht.sqlite"); + let mut import = php_base_command(layout, runtime, shared); + import + .arg( + layout + .runtime_dir + .join("scripts/cow/mysql_import_sqlite.php"), + ) + .arg(&export_path) + .arg(&db_path); + let output = import + .output() + .context("failed to run bundled MySQL-to-SQLite importer")?; + write_filtered_output(&output.stdout, &output.stderr)?; + if !output.status.success() { + bail!( + "MySQL-to-SQLite importer exited with status {}", + output.status + ); + } + let _ = fs::remove_file(&export_path); + Ok(()) +} + +fn remote_clone_ssh_command(args: &RemoteCloneArgs) -> Command { + let mut command = Command::new("ssh"); + if let Some(key) = &args.ssh_key { + command.arg("-i").arg(key); + } + if let Some(port) = args.ssh_port { + command.arg("-p").arg(port.to_string()); + } + command +} + fn remote_clone_rsync_source(ssh: &str, remote_path: &str) -> String { let mut path = remote_path.trim_end_matches('/').to_string(); path.push('/'); @@ -8326,6 +8419,44 @@ mod git_helper_tests { assert_eq!(remote_clone_rsync_ssh_command(&clone), None); } + #[test] + fn remote_clone_mysql_export_ssh_reuses_credentials() { + let clone = RemoteCloneArgs { + name: "production".to_string(), + ssh: "deploy@example.com".to_string(), + ssh_key: Some(PathBuf::from("/Users/alex/.ssh/forkpress id")), + ssh_port: Some(2222), + remote_path: "/srv/www/example with spaces".to_string(), + branch: None, + remote_url: None, + local_url: None, + include_uploads: false, + full_sync: false, + excludes: Vec::new(), + no_delete: false, + force: false, + }; + let mut command = remote_clone_ssh_command(&clone); + command + .arg(&clone.ssh) + .arg(format!("php -- {}", shell_quote(&clone.remote_path))); + let args: Vec = command + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect(); + assert_eq!( + args, + vec![ + "-i", + "/Users/alex/.ssh/forkpress id", + "-p", + "2222", + "deploy@example.com", + "php -- '/srv/www/example with spaces'", + ] + ); + } + #[test] fn registry_fields_round_trip_control_chars() { let value = "dir\\with\ttabs\nand\rreturns"; diff --git a/docs/commands.md b/docs/commands.md index 3955d3de..c465f449 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -46,7 +46,7 @@ audit queues, stale reviews, and source/target/apply-reviewed choices, see | `forkpress commit -m "message"` | Commit and push the current Git branch back to ForkPress. | | `forkpress pull` | Pull with rebase and autostash. | | `forkpress agents [dir]` | Create agent branches and Git worktrees. | -| `forkpress remote clone --ssh --ssh-key --ssh-port --path --url --branch ` | Thin-clone a boot-ready remote WordPress root over SSH, skipping uploads, caches, backups, logs, and upgrade temp files by default, then create a local COW branch. | +| `forkpress remote clone --ssh --ssh-key --ssh-port --path --url --branch ` | Thin-clone a boot-ready remote WordPress root over SSH, import MySQL into the local SQLite sidecar when needed, then create a local COW branch. | | `forkpress remote add --cache-root ` | Register an existing local remote-site cache. | | `forkpress remote branch ` | Create a local COW branch from a registered remote cache. | diff --git a/docs/remote-sites.md b/docs/remote-sites.md index f7b0616e..cf805f3c 100644 --- a/docs/remote-sites.md +++ b/docs/remote-sites.md @@ -42,8 +42,15 @@ forkpress remote clone production \ ``` The remote path must be a materialized WordPress root. It needs `wp-load.php` -and, for COW branch creation, the ForkPress-compatible SQLite database at -`wp-content/database/.ht.sqlite`. +and `wp-config.php`. + +If the remote site already has ForkPress's SQLite sidecar at +`wp-content/database/.ht.sqlite`, ForkPress uses it directly. If the remote +site is a normal MySQL-backed WordPress install, ForkPress reads the database +connection constants from `wp-config.php`, exports the WordPress tables over the +same SSH connection, and imports them into the local cache as +`wp-content/database/.ht.sqlite` before creating the branch. The remote PHP must +have `mysqli` enabled. ForkPress uses `rsync` over SSH. By default it creates a boot cache and skips large or rebuildable directories: @@ -171,9 +178,10 @@ and plugin-specific semantic recipes, see ## Troubleshooting -If branch creation reports that the source branch database does not exist, the -remote cache is not a ForkPress-compatible SQLite WordPress root. Register or -sync a cache that includes `wp-content/database/.ht.sqlite`. +If MySQL import fails, confirm that remote PHP has `mysqli`, that `DB_NAME`, +`DB_USER`, `DB_PASSWORD`, `DB_HOST`, and `$table_prefix` are string literals in +`wp-config.php`, and that the SSH user can connect to the site's MySQL database +from the remote host. If `rsync` fails, verify the SSH target, key, port, and remote path outside ForkPress first: diff --git a/scripts/cow/mysql_export.php b/scripts/cow/mysql_export.php new file mode 100644 index 00000000..035557b5 --- /dev/null +++ b/scripts/cow/mysql_export.php @@ -0,0 +1,184 @@ + + */ + +declare(strict_types=1); + +error_reporting(E_ALL); + +$root = rtrim((string)($argv[1] ?? ''), "/\\"); +if ($root === '' || !is_dir($root)) { + fwrite(STDERR, "mysql_export: WordPress root missing or not a directory\n"); + exit(2); +} + +$config_path = $root . '/wp-config.php'; +if (!is_file($config_path)) { + fwrite(STDERR, "mysql_export: wp-config.php not found in $root\n"); + exit(2); +} + +if (!extension_loaded('mysqli')) { + fwrite(STDERR, "mysql_export: remote PHP is missing mysqli\n"); + exit(2); +} + +$config = file_get_contents($config_path); +if ($config === false) { + fwrite(STDERR, "mysql_export: failed to read $config_path\n"); + exit(2); +} + +function forkpress_mysql_config_define(string $config, string $name): string { + $pattern = '/define\s*\(\s*[\'"]' . preg_quote($name, '/') . '[\'"]\s*,\s*([\'"])(.*?)\1\s*\)/s'; + if (!preg_match($pattern, $config, $matches)) { + throw new RuntimeException("wp-config.php must define $name as a string literal"); + } + return stripcslashes($matches[2]); +} + +function forkpress_mysql_table_prefix(string $config): string { + if (!preg_match('/\$table_prefix\s*=\s*([\'"])(.*?)\1\s*;/s', $config, $matches)) { + throw new RuntimeException('wp-config.php must assign $table_prefix as a string literal'); + } + return stripcslashes($matches[2]); +} + +function forkpress_mysql_quote_identifier(string $identifier): string { + return '`' . str_replace('`', '``', $identifier) . '`'; +} + +function forkpress_mysql_emit(array $record): void { + $json = json_encode($record, JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_SUBSTITUTE); + if ($json === false) { + throw new RuntimeException('failed to encode export record: ' . json_last_error_msg()); + } + echo $json, "\n"; +} + +try { + $db_name = forkpress_mysql_config_define($config, 'DB_NAME'); + $db_user = forkpress_mysql_config_define($config, 'DB_USER'); + $db_password = forkpress_mysql_config_define($config, 'DB_PASSWORD'); + $db_host_raw = forkpress_mysql_config_define($config, 'DB_HOST'); + $table_prefix = forkpress_mysql_table_prefix($config); +} catch (Throwable $e) { + fwrite(STDERR, 'mysql_export: ' . $e->getMessage() . "\n"); + exit(2); +} + +$host = $db_host_raw === '' ? 'localhost' : $db_host_raw; +$port = 0; +$socket = null; +if (strpos($host, ':/') !== false) { + [$host, $socket] = explode(':', $host, 2); +} elseif (preg_match('/^(.+):([0-9]+)$/', $host, $matches)) { + $host = $matches[1]; + $port = (int)$matches[2]; +} + +$mysqli = mysqli_init(); +if (!$mysqli) { + fwrite(STDERR, "mysql_export: failed to initialize mysqli\n"); + exit(2); +} +if (!$mysqli->real_connect($host, $db_user, $db_password, $db_name, $port, $socket)) { + fwrite(STDERR, 'mysql_export: MySQL connection failed: ' . mysqli_connect_error() . "\n"); + exit(2); +} +$mysqli->set_charset('utf8mb4'); + +forkpress_mysql_emit([ + 'type' => 'meta', + 'database' => $db_name, + 'table_prefix' => $table_prefix, +]); + +$tables = []; +$result = $mysqli->query('SHOW FULL TABLES'); +if (!$result) { + fwrite(STDERR, 'mysql_export: failed to list tables: ' . $mysqli->error . "\n"); + exit(2); +} +while ($row = $result->fetch_array(MYSQLI_NUM)) { + $table = (string)($row[0] ?? ''); + $kind = strtoupper((string)($row[1] ?? 'BASE TABLE')); + if ($kind !== 'BASE TABLE' || strncmp($table, $table_prefix, strlen($table_prefix)) !== 0) { + continue; + } + $tables[] = $table; +} +$result->free(); +sort($tables, SORT_STRING); + +foreach ($tables as $table) { + $quoted = forkpress_mysql_quote_identifier($table); + $columns = []; + $column_result = $mysqli->query("SHOW COLUMNS FROM $quoted"); + if (!$column_result) { + throw new RuntimeException("failed to read columns for $table: " . $mysqli->error); + } + while ($column = $column_result->fetch_assoc()) { + $columns[] = [ + 'name' => (string)$column['Field'], + 'type' => (string)$column['Type'], + 'null' => (string)$column['Null'], + 'key' => (string)$column['Key'], + 'default' => $column['Default'], + 'extra' => (string)$column['Extra'], + ]; + } + $column_result->free(); + + $indexes = []; + $index_result = $mysqli->query("SHOW INDEX FROM $quoted"); + if (!$index_result) { + throw new RuntimeException("failed to read indexes for $table: " . $mysqli->error); + } + while ($index = $index_result->fetch_assoc()) { + $key = (string)$index['Key_name']; + if ($key === 'PRIMARY') { + continue; + } + $indexes[] = [ + 'name' => $key, + 'unique' => ((int)$index['Non_unique']) === 0, + 'seq' => (int)$index['Seq_in_index'], + 'column' => $index['Column_name'] === null ? null : (string)$index['Column_name'], + 'sub_part' => $index['Sub_part'] === null ? null : (int)$index['Sub_part'], + ]; + } + $index_result->free(); + + forkpress_mysql_emit([ + 'type' => 'table', + 'name' => $table, + 'columns' => $columns, + 'indexes' => $indexes, + ]); + + $row_result = $mysqli->query("SELECT * FROM $quoted", MYSQLI_USE_RESULT); + if (!$row_result) { + throw new RuntimeException("failed to read rows for $table: " . $mysqli->error); + } + while ($row = $row_result->fetch_assoc()) { + $values = []; + foreach ($row as $name => $value) { + $values[$name] = $value === null ? null : base64_encode((string)$value); + } + forkpress_mysql_emit([ + 'type' => 'row', + 'table' => $table, + 'values' => $values, + ]); + } + $row_result->free(); +} diff --git a/scripts/cow/mysql_import_sqlite.php b/scripts/cow/mysql_import_sqlite.php new file mode 100644 index 00000000..4c6ee959 --- /dev/null +++ b/scripts/cow/mysql_import_sqlite.php @@ -0,0 +1,264 @@ + + */ + +declare(strict_types=1); + +error_reporting(E_ALL); + +$export_path = (string)($argv[1] ?? ''); +$db_path = (string)($argv[2] ?? ''); +if ($export_path === '' || !is_file($export_path)) { + fwrite(STDERR, "mysql_import_sqlite: export file missing\n"); + exit(2); +} +if ($db_path === '') { + fwrite(STDERR, "mysql_import_sqlite: SQLite database path missing\n"); + exit(2); +} +if (!class_exists('SQLite3')) { + fwrite(STDERR, "mysql_import_sqlite: local PHP is missing SQLite3\n"); + exit(2); +} + +function forkpress_mysql_import_ident(string $name): string { + if (!preg_match('/^[A-Za-z0-9_]+$/', $name)) { + throw new RuntimeException("unsupported MySQL identifier: $name"); + } + return '"' . str_replace('"', '""', $name) . '"'; +} + +function forkpress_mysql_import_affinity(string $mysql_type): string { + $type = strtolower($mysql_type); + if (str_contains($type, 'blob') || str_contains($type, 'binary')) { + return 'BLOB'; + } + if (str_contains($type, 'int') || str_contains($type, 'bit') || str_contains($type, 'bool')) { + return 'INTEGER'; + } + if (str_contains($type, 'decimal') || str_contains($type, 'double') || str_contains($type, 'float') || str_contains($type, 'real')) { + return 'REAL'; + } + return 'TEXT'; +} + +function forkpress_mysql_import_decode_value(mixed $encoded): ?string { + if ($encoded === null) { + return null; + } + if (!is_string($encoded)) { + throw new RuntimeException('row value must be null or a base64 string'); + } + $decoded = base64_decode($encoded, true); + if ($decoded === false) { + throw new RuntimeException('row value is not valid base64'); + } + return $decoded; +} + +function forkpress_mysql_import_bind(SQLite3Stmt $stmt, string $param, ?string $value, string $mysql_type): void { + if ($value === null) { + $stmt->bindValue($param, null, SQLITE3_NULL); + return; + } + $affinity = forkpress_mysql_import_affinity($mysql_type); + if ($affinity === 'INTEGER' && preg_match('/^-?[0-9]+$/', $value)) { + $stmt->bindValue($param, (int)$value, SQLITE3_INTEGER); + return; + } + if ($affinity === 'REAL' && is_numeric($value)) { + $stmt->bindValue($param, (float)$value, SQLITE3_FLOAT); + return; + } + if ($affinity === 'BLOB') { + $stmt->bindValue($param, $value, SQLITE3_BLOB); + return; + } + $stmt->bindValue($param, $value, SQLITE3_TEXT); +} + +function forkpress_mysql_import_create_table(SQLite3 $db, array $table): array { + $name = (string)($table['name'] ?? ''); + $columns = $table['columns'] ?? null; + if ($name === '' || !is_array($columns) || count($columns) === 0) { + throw new RuntimeException('table record must include name and columns'); + } + + $primary = []; + $auto_primary = null; + $column_types = []; + foreach ($columns as $column) { + $column_name = (string)($column['name'] ?? ''); + $key = strtoupper((string)($column['key'] ?? '')); + $extra = strtolower((string)($column['extra'] ?? '')); + if ($column_name === '') { + throw new RuntimeException("table $name has an unnamed column"); + } + $column_types[$column_name] = (string)($column['type'] ?? 'text'); + if ($key === 'PRI') { + $primary[] = $column_name; + if (str_contains($extra, 'auto_increment')) { + $auto_primary = $column_name; + } + } + } + + $defs = []; + foreach ($columns as $column) { + $column_name = (string)$column['name']; + $affinity = forkpress_mysql_import_affinity((string)($column['type'] ?? 'text')); + if ($auto_primary === $column_name && count($primary) === 1 && $affinity === 'INTEGER') { + $defs[] = forkpress_mysql_import_ident($column_name) . ' INTEGER PRIMARY KEY AUTOINCREMENT'; + } else { + $defs[] = forkpress_mysql_import_ident($column_name) . ' ' . $affinity; + } + } + if ($auto_primary === null && count($primary) > 0) { + $defs[] = 'PRIMARY KEY (' . implode(', ', array_map('forkpress_mysql_import_ident', $primary)) . ')'; + } + + $db->exec('DROP TABLE IF EXISTS ' . forkpress_mysql_import_ident($name)); + $sql = 'CREATE TABLE ' . forkpress_mysql_import_ident($name) . ' (' . implode(', ', $defs) . ')'; + if (!$db->exec($sql)) { + throw new RuntimeException("failed to create SQLite table $name: " . $db->lastErrorMsg()); + } + + $indexes = $table['indexes'] ?? []; + if (is_array($indexes)) { + $grouped = []; + foreach ($indexes as $index) { + $index_name = (string)($index['name'] ?? ''); + $column = $index['column'] ?? null; + if ($index_name === '' || $index_name === 'PRIMARY' || $column === null) { + continue; + } + $grouped[$index_name]['unique'] = (bool)($index['unique'] ?? false); + $grouped[$index_name]['columns'][(int)($index['seq'] ?? 0)] = (string)$column; + } + foreach ($grouped as $index_name => $index) { + ksort($index['columns'], SORT_NUMERIC); + $columns_sql = implode(', ', array_map('forkpress_mysql_import_ident', $index['columns'])); + if ($columns_sql === '') { + continue; + } + $sqlite_index = $name . '__' . $index_name; + $sql = 'CREATE ' . ($index['unique'] ? 'UNIQUE ' : '') . 'INDEX ' + . forkpress_mysql_import_ident($sqlite_index) + . ' ON ' . forkpress_mysql_import_ident($name) + . ' (' . $columns_sql . ')'; + if (!$db->exec($sql)) { + throw new RuntimeException("failed to create SQLite index $sqlite_index: " . $db->lastErrorMsg()); + } + } + } + + return $column_types; +} + +$parent = dirname($db_path); +if (!is_dir($parent) && !mkdir($parent, 0755, true) && !is_dir($parent)) { + fwrite(STDERR, "mysql_import_sqlite: failed to create $parent\n"); + exit(2); +} +$tmp_path = $db_path . '.importing.' . getmypid(); +@unlink($tmp_path); + +$tables = []; +$prepared = []; +$table_count = 0; +$row_count = 0; + +try { + $db = new SQLite3($tmp_path); + $db->busyTimeout(5000); + $db->exec('PRAGMA foreign_keys = OFF'); + $db->exec('PRAGMA journal_mode = WAL'); + $db->exec('BEGIN IMMEDIATE'); + + $handle = fopen($export_path, 'rb'); + if (!$handle) { + throw new RuntimeException("failed to open $export_path"); + } + while (($line = fgets($handle)) !== false) { + $line = trim($line); + if ($line === '') { + continue; + } + $record = json_decode($line, true); + if (!is_array($record)) { + throw new RuntimeException('invalid JSON export record: ' . json_last_error_msg()); + } + $type = (string)($record['type'] ?? ''); + if ($type === 'meta') { + continue; + } + if ($type === 'table') { + $table = (string)($record['name'] ?? ''); + $tables[$table] = forkpress_mysql_import_create_table($db, $record); + $table_count++; + continue; + } + if ($type !== 'row') { + throw new RuntimeException("unsupported export record type: $type"); + } + + $table = (string)($record['table'] ?? ''); + if (!isset($tables[$table])) { + throw new RuntimeException("row appeared before table metadata for $table"); + } + $values = $record['values'] ?? null; + if (!is_array($values)) { + throw new RuntimeException("row for $table must include values"); + } + $columns = array_keys($tables[$table]); + if (!isset($prepared[$table])) { + $placeholders = []; + foreach ($columns as $i => $_column) { + $placeholders[] = ':v' . $i; + } + $sql = 'INSERT INTO ' . forkpress_mysql_import_ident($table) + . ' (' . implode(', ', array_map('forkpress_mysql_import_ident', $columns)) . ')' + . ' VALUES (' . implode(', ', $placeholders) . ')'; + $stmt = $db->prepare($sql); + if (!$stmt) { + throw new RuntimeException("failed to prepare insert for $table: " . $db->lastErrorMsg()); + } + $prepared[$table] = $stmt; + } + $stmt = $prepared[$table]; + foreach ($columns as $i => $column) { + $decoded = forkpress_mysql_import_decode_value($values[$column] ?? null); + forkpress_mysql_import_bind($stmt, ':v' . $i, $decoded, $tables[$table][$column]); + } + $result = $stmt->execute(); + if (!$result) { + throw new RuntimeException("failed to insert row into $table: " . $db->lastErrorMsg()); + } + $result->finalize(); + $stmt->reset(); + $row_count++; + } + fclose($handle); + + $db->exec('COMMIT'); + $db->close(); + if (is_file($db_path) && !unlink($db_path)) { + throw new RuntimeException("failed to replace existing SQLite database at $db_path"); + } + if (!rename($tmp_path, $db_path)) { + throw new RuntimeException("failed to publish SQLite database at $db_path"); + } + echo " mysql: imported $table_count tables, $row_count rows into wp-content/database/.ht.sqlite\n"; +} catch (Throwable $e) { + if (isset($db) && $db instanceof SQLite3) { + @$db->exec('ROLLBACK'); + @$db->close(); + } + @unlink($tmp_path); + fwrite(STDERR, 'mysql_import_sqlite: ' . $e->getMessage() . "\n"); + exit(2); +} diff --git a/tests/cow/e2e.sh b/tests/cow/e2e.sh index 74c0ef6d..75410bcb 100755 --- a/tests/cow/e2e.sh +++ b/tests/cow/e2e.sh @@ -1415,6 +1415,89 @@ test ! -e "$WORK/remote-thin-branch/wp-content/uploads/2026/05/large-upload.jpg" autoinc_runtime_request remote-thin-branch insert "$TMP/autoinc-remote-thin-insert.json" php -r '$data = json_decode(file_get_contents($argv[1]), true); $meta = new SQLite3($argv[2]); $branch = new SQLite3($argv[3]); $max = (int)($data["max_id"] ?? 0); $band = $meta->querySingle("SELECT band_start, band_end FROM merge_autoincrement_bands WHERE branch_name = '\''remote-thin-branch'\'' AND table_name = '\''wp_forkpress_e2e_autoinc'\''", true); $has_db_base = is_file($argv[4]); $has_file_base = is_file($argv[5]); $seq = (int)$branch->querySingle("SELECT seq FROM sqlite_sequence WHERE name = '\''wp_forkpress_e2e_autoinc'\''"); exit($band && $has_db_base && $has_file_base && $max >= (int)$band["band_start"] && $max <= (int)$band["band_end"] && $seq === $max ? 0 : 1);' "$TMP/autoinc-remote-thin-insert.json" "$WORK_DIR/cow/merge/metadata.sqlite" "$WORK/remote-thin-branch/wp-content/database/.ht.sqlite" "$WORK_DIR/cow/merge/bases/remote-thin-branch.sqlite" "$WORK_DIR/cow/merge/file-bases/remote-thin-branch.json" +log_step "remote clone imports MySQL-backed WordPress cache before branching" +REMOTE_MYSQL_SOURCE="$TMP/remote-mysql-source" +mkdir -p "$REMOTE_MYSQL_SOURCE" +cp -R "$WORK/main/." "$REMOTE_MYSQL_SOURCE/" +rm -rf "$REMOTE_MYSQL_SOURCE/wp-content/database" +cat > "$REMOTE_MYSQL_SOURCE/wp-config.php" <<'PHP' + "$FAKE_RSYNC_BIN/ssh" <<'SH' +#!/usr/bin/env bash +set -euo pipefail +cat >/dev/null +php -r ' +function emit($record) { echo json_encode($record, JSON_UNESCAPED_SLASHES), "\n"; } +emit(["type" => "meta", "database" => "wordpress", "table_prefix" => "wp_"]); +emit([ + "type" => "table", + "name" => "wp_posts", + "columns" => [ + ["name" => "ID", "type" => "bigint(20) unsigned", "null" => "NO", "key" => "PRI", "default" => null, "extra" => "auto_increment"], + ["name" => "post_title", "type" => "text", "null" => "NO", "key" => "", "default" => null, "extra" => ""], + ["name" => "post_status", "type" => "varchar(20)", "null" => "NO", "key" => "", "default" => "publish", "extra" => ""] + ], + "indexes" => [ + ["name" => "post_status", "unique" => false, "seq" => 1, "column" => "post_status"] + ] +]); +emit([ + "type" => "row", + "table" => "wp_posts", + "values" => [ + "ID" => base64_encode("11"), + "post_title" => base64_encode("Imported remote MySQL page"), + "post_status" => base64_encode("publish") + ] +]); +emit([ + "type" => "table", + "name" => "wp_options", + "columns" => [ + ["name" => "option_id", "type" => "bigint(20) unsigned", "null" => "NO", "key" => "PRI", "default" => null, "extra" => "auto_increment"], + ["name" => "option_name", "type" => "varchar(191)", "null" => "NO", "key" => "UNI", "default" => "", "extra" => ""], + ["name" => "option_value", "type" => "longtext", "null" => "NO", "key" => "", "default" => null, "extra" => ""], + ["name" => "autoload", "type" => "varchar(20)", "null" => "NO", "key" => "", "default" => "yes", "extra" => ""] + ], + "indexes" => [ + ["name" => "option_name", "unique" => true, "seq" => 1, "column" => "option_name"] + ] +]); +emit([ + "type" => "row", + "table" => "wp_options", + "values" => [ + "option_id" => base64_encode("1"), + "option_name" => base64_encode("siteurl"), + "option_value" => base64_encode("https://mysql.example.test"), + "autoload" => base64_encode("yes") + ] +]); +' +SH +chmod +x "$FAKE_RSYNC_BIN/ssh" +PATH="$FAKE_RSYNC_BIN:$PATH" "$BIN" remote --work-dir "$WORK_DIR" clone mysql-prod \ + --ssh fake-mysql-remote \ + --path "$REMOTE_MYSQL_SOURCE" \ + --branch remote-mysql-branch \ + --remote-url "https://mysql.example.test/" \ + > "$TMP/remote-mysql-clone.out" +grep -F "forkpress: remote site 'mysql-prod' cloned" "$TMP/remote-mysql-clone.out" >/dev/null +grep -F "mysql: imported 2 tables, 2 rows into wp-content/database/.ht.sqlite" "$TMP/remote-mysql-clone.out" >/dev/null +grep -F "forkpress: remote cache 'mysql-prod' branched to 'remote-mysql-branch'" "$TMP/remote-mysql-clone.out" >/dev/null +test -f "$WORK_DIR/cow/remote-sites/mysql-prod/cache/wp-content/database/.ht.sqlite" +test -f "$WORK/remote-mysql-branch/wp-content/database/.ht.sqlite" +test -f "$WORK_DIR/cow/merge/bases/remote-mysql-branch.sqlite" +test -f "$WORK_DIR/cow/merge/file-bases/remote-mysql-branch.json" +php -r '$db = new SQLite3($argv[1]); $title = $db->querySingle("SELECT post_title FROM wp_posts WHERE ID = 11"); $seq = (int)$db->querySingle("SELECT seq FROM sqlite_sequence WHERE name = '\''wp_posts'\''"); exit($title === "Imported remote MySQL page" && $seq >= 11 ? 0 : 1);' "$WORK/remote-mysql-branch/wp-content/database/.ht.sqlite" +php -r '$meta = new SQLite3($argv[1]); $band = $meta->querySingle("SELECT band_start, band_end FROM merge_autoincrement_bands WHERE branch_name = '\''remote-mysql-branch'\'' AND table_name = '\''wp_posts'\''", true); exit($band && (int)$band["band_start"] > 11 ? 0 : 1);' "$WORK_DIR/cow/merge/metadata.sqlite" + if [ "${FORKPRESS_E2E_ONLY:-}" = "remote-cache" ]; then log_step "remote cache branch slice complete" exit 0 diff --git a/tests/cow/mysql_import.php b/tests/cow/mysql_import.php new file mode 100644 index 00000000..39fd14cc --- /dev/null +++ b/tests/cow/mysql_import.php @@ -0,0 +1,109 @@ +isDir() ? rmdir($file->getPathname()) : unlink($file->getPathname()); + } + rmdir($path); +} + +function emit_record($handle, array $record): void { + fwrite($handle, json_encode($record, JSON_UNESCAPED_SLASHES) . "\n"); +} + +$export = $tmp . '/mysql.jsonl'; +$db_path = $tmp . '/wp-content/database/.ht.sqlite'; +$fh = fopen($export, 'wb'); +emit_record($fh, ['type' => 'meta', 'database' => 'wordpress', 'table_prefix' => 'wp_']); +emit_record($fh, [ + 'type' => 'table', + 'name' => 'wp_posts', + 'columns' => [ + ['name' => 'ID', 'type' => 'bigint(20) unsigned', 'null' => 'NO', 'key' => 'PRI', 'default' => null, 'extra' => 'auto_increment'], + ['name' => 'post_title', 'type' => 'text', 'null' => 'NO', 'key' => '', 'default' => null, 'extra' => ''], + ['name' => 'post_status', 'type' => 'varchar(20)', 'null' => 'NO', 'key' => '', 'default' => 'publish', 'extra' => ''], + ], + 'indexes' => [ + ['name' => 'post_status', 'unique' => false, 'seq' => 1, 'column' => 'post_status'], + ], +]); +emit_record($fh, [ + 'type' => 'row', + 'table' => 'wp_posts', + 'values' => [ + 'ID' => base64_encode('7'), + 'post_title' => base64_encode('Imported MySQL page'), + 'post_status' => base64_encode('publish'), + ], +]); +emit_record($fh, [ + 'type' => 'table', + 'name' => 'wp_options', + 'columns' => [ + ['name' => 'option_id', 'type' => 'bigint(20) unsigned', 'null' => 'NO', 'key' => 'PRI', 'default' => null, 'extra' => 'auto_increment'], + ['name' => 'option_name', 'type' => 'varchar(191)', 'null' => 'NO', 'key' => 'UNI', 'default' => '', 'extra' => ''], + ['name' => 'option_value', 'type' => 'longtext', 'null' => 'NO', 'key' => '', 'default' => null, 'extra' => ''], + ['name' => 'autoload', 'type' => 'varchar(20)', 'null' => 'NO', 'key' => '', 'default' => 'yes', 'extra' => ''], + ], + 'indexes' => [ + ['name' => 'option_name', 'unique' => true, 'seq' => 1, 'column' => 'option_name'], + ], +]); +emit_record($fh, [ + 'type' => 'row', + 'table' => 'wp_options', + 'values' => [ + 'option_id' => base64_encode('3'), + 'option_name' => base64_encode('siteurl'), + 'option_value' => base64_encode('https://example.test'), + 'autoload' => base64_encode('yes'), + ], +]); +fclose($fh); + +$cmd = escapeshellarg(PHP_BINARY) . ' ' + . escapeshellarg(__DIR__ . '/../../scripts/cow/mysql_import_sqlite.php') . ' ' + . escapeshellarg($export) . ' ' + . escapeshellarg($db_path); +passthru($cmd, $status); +if ($status !== 0) { + cleanup($tmp); + exit($status); +} + +$db = new SQLite3($db_path, SQLITE3_OPEN_READWRITE); +$title = $db->querySingle("SELECT post_title FROM wp_posts WHERE ID = 7"); +$sequence = (int)$db->querySingle("SELECT seq FROM sqlite_sequence WHERE name = 'wp_posts'"); +$index = $db->querySingle("SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'wp_options__option_name'"); +$duplicate_failed = false; +try { + $duplicate_failed = !@$db->exec("INSERT INTO wp_options (option_id, option_name, option_value, autoload) VALUES (4, 'siteurl', 'duplicate', 'yes')"); +} catch (Throwable $e) { + $duplicate_failed = true; +} +$db->close(); + +$ok = $title === 'Imported MySQL page' + && $sequence === 7 + && $index === 'wp_options__option_name' + && $duplicate_failed; +cleanup($tmp); +if (!$ok) { + fwrite(STDERR, "mysql import assertions failed\n"); + exit(1); +} +echo "mysql_import.php passed\n";