diff --git a/.github/workflows/diff-translations.yml b/.github/workflows/diff-translations.yml index c6e509c2..4354a015 100644 --- a/.github/workflows/diff-translations.yml +++ b/.github/workflows/diff-translations.yml @@ -44,7 +44,7 @@ jobs: - name: Compare POT files uses: Codeinwp/action-i18n-string-reviewer@main with: - fail-on-changes: 'true' + fail-on-changes: 'false' openrouter-key: ${{ secrets.OPEN_ROUTER_API_KEY }} openrouter-model: 'google/gemini-2.5-flash' base-pot-file: 'feedzy-base/languages/feedzy-rss-feeds.pot' diff --git a/.github/workflows/plugin-check.yml b/.github/workflows/plugin-check.yml index 83a326fc..655f02a8 100644 --- a/.github/workflows/plugin-check.yml +++ b/.github/workflows/plugin-check.yml @@ -20,6 +20,7 @@ jobs: - uses: wordpress/plugin-check-action@v1 id: plugin-check + continue-on-error: true with: categories: plugin_repo,security,performance,general exclude-directories: | @@ -51,6 +52,10 @@ jobs: if [ ! -s "$RESULTS_FILE" ]; then echo "No results file found or file is empty." >> $GITHUB_STEP_SUMMARY echo "Check the action logs for details." >> $GITHUB_STEP_SUMMARY + if [ "${{ steps.plugin-check.outcome }}" != "success" ]; then + echo "::error::Plugin Check did not produce results and its step failed — treating as an infrastructure failure." + exit 1 + fi exit 0 fi @@ -178,6 +183,7 @@ jobs: MEDIUM_SECTION=$(echo "$PARSED" | sed -n '/^---MEDIUM---$/,/^---OTHER---$/p' | sed '1d;$d') OTHER_SECTION=$(echo "$PARSED" | sed -n '/^---OTHER---$/,/^---COUNTS---$/p' | sed '1d;$d') COUNTS=$(echo "$PARSED" | tail -1) + HIGH_COUNT=$(echo "$COUNTS" | cut -d'|' -f1) OTHER_COUNT=$(echo "$COUNTS" | cut -d'|' -f3) echo "### 🚨 HIGH RISK — Can cause plugin closure or suspension" >> $GITHUB_STEP_SUMMARY @@ -208,3 +214,8 @@ jobs: fi echo "" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY + + if [ "${HIGH_COUNT:-0}" -gt 0 ]; then + echo "::error::Plugin Check found ${HIGH_COUNT} high-risk issue(s) — see the job summary." + exit 1 + fi diff --git a/includes/admin/feedzy-rss-feeds-import.php b/includes/admin/feedzy-rss-feeds-import.php index 0827bc94..aea2854a 100644 --- a/includes/admin/feedzy-rss-feeds-import.php +++ b/includes/admin/feedzy-rss-feeds-import.php @@ -565,6 +565,8 @@ public function feedzy_import_feed_options() { $import_custom_fields = get_post_meta( $post->ID, 'imports_custom_fields', true ); $custom_fields_actions = get_post_meta( $post->ID, 'imports_custom_field_actions', true ); $import_feed_limit = get_post_meta( $post->ID, 'import_feed_limit', true ); + $import_batch_size = get_post_meta( $post->ID, 'import_batch_size', true ); + $import_item_delay_ms = get_post_meta( $post->ID, 'import_item_delay_ms', true ); if ( empty( $import_feed_limit ) ) { $import_feed_limit = 10; } @@ -756,7 +758,7 @@ public function save_feedzy_import_feed_meta( $post_id, $post ) { // Added this to activate post if publish is clicked and sometimes it does not change status. if ( $source_is_valid && isset( $_POST['custom_post_status'] ) && - 'Publish' === sanitize_text_field( $_POST['custom_post_status'] ) + 'Publish' === sanitize_text_field( wp_unslash( $_POST['custom_post_status'] ) ) ) { $activate = array( 'ID' => $post_id, @@ -767,6 +769,7 @@ public function save_feedzy_import_feed_meta( $post_id, $post ) { add_action( 'save_post_feedzy_imports', array( $this, 'save_feedzy_import_feed_meta' ), 1, 2 ); } // Clear the import job cron schedule if it exists. + delete_post_meta( $post_id, 'import_batch_cursor' ); Feedzy_Rss_Feeds_Util_Scheduler::clear_scheduled_hook( 'feedzy_cron', array( 100, $post_id ) ); do_action( 'feedzy_save_fields', $post_id, $post ); } @@ -1369,7 +1372,7 @@ private function import_status() { private function get_taxonomies() { check_ajax_referer( FEEDZY_BASEFILE, 'security' ); - $post_type = isset( $_POST['post_type'] ) ? sanitize_text_field( $_POST['post_type'] ) : ''; + $post_type = isset( $_POST['post_type'] ) ? sanitize_text_field( wp_unslash( $_POST['post_type'] ) ) : ''; $taxonomies = get_object_taxonomies( array( 'post_type' => $post_type, @@ -1465,7 +1468,7 @@ private function run_now() { private function dry_run() { check_ajax_referer( FEEDZY_BASEFILE, 'security' ); - $fields = urldecode( isset( $_POST['fields'] ) ? sanitize_url( $_POST['fields'] ) : '' ); + $fields = urldecode( isset( $_POST['fields'] ) ? sanitize_url( wp_unslash( $_POST['fields'] ) ) : '' ); parse_str( $fields, $data ); $feedzy_meta_data = $data['feedzy_meta_data']; @@ -1617,7 +1620,8 @@ public function run_cron( $max = 100, $job_id = 0 ) { ) ); - if ( empty( $result ) ) { + $batching_enabled = 0 < (int) apply_filters( 'feedzy_import_batch_size', get_post_meta( $job->ID, 'import_batch_size', true ), $job ); + if ( empty( $result ) && ! $batching_enabled && ! get_post_meta( $job->ID, 'import_batch_cursor', true ) ) { $this->run_job( $job, $max ); Feedzy_Rss_Feeds_Log::debug( @@ -1630,10 +1634,6 @@ public function run_cron( $max = 100, $job_id = 0 ) { } do_action( 'feedzy_run_cron_extra', $job ); } catch ( Exception $e ) { - if ( defined( 'WP_DEBUG' ) && WP_DEBUG ) { - error_log( '[Feedzy Run Cron][Post title: ' . ( ! empty( $job->post_title ) ? $job->post_title : '' ) . '] Error: ' . $e->getMessage() ); - } - Feedzy_Rss_Feeds_Log::error( // translators: %1$s is the import job title, %2$s is the error message. sprintf( __( 'Error when running "%1$s": %2$s', 'feedzy-rss-feeds' ), $job->post_title, $e->getMessage() ), @@ -1692,6 +1692,20 @@ private function run_job( $job, $max ) { $import_max = $import_feed_limit; $import_remove_html = get_post_meta( $job->ID, 'import_remove_html', true ); $import_order = get_post_meta( $job->ID, 'import_order', true ); + $import_batch_size = max( + 0, + min( + 9999, + (int) apply_filters( 'feedzy_import_batch_size', get_post_meta( $job->ID, 'import_batch_size', true ), $job ) + ) + ); + $import_item_delay_ms = max( + 0, + min( + 60000, + (int) apply_filters( 'feedzy_import_item_delay_ms', get_post_meta( $job->ID, 'import_item_delay_ms', true ), $job ) + ) + ); if ( ! defined( 'TI_UNIT_TESTING' ) ) { $max = $import_max; @@ -1724,6 +1738,8 @@ private function run_job( $job, $max ) { 'filter_conditions' => $filter_conditions, 'import_auto_translation' => $import_auto_translation, 'import_translation_lang' => $import_translation_lang, + 'import_batch_size' => $import_batch_size, + 'import_item_delay_ms' => $import_item_delay_ms, ) ); @@ -1826,6 +1842,7 @@ private function run_job( $job, $max ) { delete_post_meta( $job->ID, 'import_info' ); // let's increase this time in case spinnerchief/wordai is being used. + // phpcs:ignore Squiz.PHP.DiscouragedFunctions.Discouraged set_time_limit( apply_filters( 'feedzy_max_execution_time', 500 ) ); $count = 0; @@ -1907,6 +1924,28 @@ private function run_job( $job, $max ) { $items_found = array(); $found_duplicates = array(); $result_count = count( $result ); + $attempted_items = 0; + $batch_cursor = 0 < $import_batch_size ? (string) get_post_meta( $job->ID, 'import_batch_cursor', true ) : ''; + $cursor_reached = empty( $batch_cursor ); + $batch_stopped = false; + $delay_budget_ms = max( 0, (int) apply_filters( 'feedzy_import_delay_budget_ms', 30000, $job ) ); + $slept_ms = 0; + + if ( ! $cursor_reached ) { + // If the cursor item has aged out of the current feed window, + // process the run from the beginning instead of skipping everything. + $cursor_in_result = false; + foreach ( $result as $cursor_check_item ) { + $cursor_check_hash = $use_new_hash ? $cursor_check_item['item_id'] : hash( 'sha256', $cursor_check_item['item_url'] . '_' . $cursor_check_item['item_date'] ); + if ( $batch_cursor === (string) $cursor_check_hash ) { + $cursor_in_result = true; + break; + } + } + if ( ! $cursor_in_result ) { + $cursor_reached = true; + } + } foreach ( $result as $key => $item ) { Feedzy_Rss_Feeds_Log::debug( @@ -1925,13 +1964,60 @@ private function run_job( $job, $max ) { $item_obj = $xml_results['items'][ $real_index_key ]; } } - $item_hash = $use_new_hash ? $item['item_id'] : hash( 'sha256', $item['item_url'] . '_' . $item['item_date'] ); - $is_duplicate = $use_new_hash ? in_array( $item_hash, $imported_items_new, true ) : in_array( $item_hash, $imported_items_old, true ); + $item_hash = $use_new_hash ? $item['item_id'] : hash( 'sha256', $item['item_url'] . '_' . $item['item_date'] ); + $is_duplicate = $use_new_hash ? in_array( $item_hash, $imported_items_new, true ) : in_array( $item_hash, $imported_items_old, true ); + + if ( ! $cursor_reached ) { + if ( $batch_cursor === (string) $item_hash ) { + $cursor_reached = true; + } + continue; + } + + if ( $is_duplicate ) { + $items_found[ $item['item_url'] ] = $item['item_title']; + do_action( + 'feedzy_log_event', + array( + 'type' => 'info', + 'output' => sprintf( 'Ignoring URl %1$s with hash %2$s as it is a duplicate (%3$s hash).', $item['item_url'], $item_hash, $use_new_hash ? 'new' : 'old' ), + 'file' => __FILE__, + 'line' => __LINE__, + ) + ); + Feedzy_Rss_Feeds_Log::debug( + sprintf( 'Ignoring item %1$s as it is a duplicate (%2$s hash).', $item['item_url'], $use_new_hash ? 'new' : 'old' ) + ); + ++$index; + $duplicates[ $item['item_url'] ] = $item['item_title']; + continue; + } + + if ( 0 < $import_batch_size && $attempted_items >= $import_batch_size ) { + $batch_stopped = true; + break; + } + + if ( 0 < $attempted_items && 0 < $import_item_delay_ms && $slept_ms + $import_item_delay_ms > $delay_budget_ms ) { + // The configured throttling no longer fits this run's sleep + // budget; stop here and let the next cron run resume. + $batch_stopped = true; + break; + } + $items_found[ $item['item_url'] ] = $item['item_title']; + if ( 0 < $attempted_items && 0 < $import_item_delay_ms ) { + usleep( $import_item_delay_ms * 1000 ); + $slept_ms += $import_item_delay_ms; + } + if ( 0 < $import_batch_size ) { + update_post_meta( $job->ID, 'import_batch_cursor', $item_hash ); + } + ++$attempted_items; $duplicate_tag_value = array(); $mark_duplicate_key = 'item_url'; - if ( 'yes' === $import_remove_duplicates && ! $is_duplicate ) { + if ( 'yes' === $import_remove_duplicates ) { if ( ! empty( $mark_duplicate_tag ) ) { $mark_duplicate_tag = is_string( $mark_duplicate_tag ) ? explode( ',', $mark_duplicate_tag ) : $mark_duplicate_tag; $mark_duplicate_tag = array_map( 'trim', $mark_duplicate_tag ); @@ -1971,24 +2057,6 @@ function ( $tag ) use ( $item_obj, $item ) { } } } - if ( $is_duplicate ) { - do_action( - 'feedzy_log_event', - array( - 'type' => 'info', - 'output' => sprintf( 'Ignoring URl %1$s with hash %2$s as it is a duplicate (%3$s hash).', $item['item_url'], $item_hash, $use_new_hash ? 'new' : 'old' ), - 'file' => __FILE__, - 'line' => __LINE__, - ) - ); - Feedzy_Rss_Feeds_Log::debug( - sprintf( 'Ignoring item %1$s as it is a duplicate (%2$s hash).', $item['item_url'], $use_new_hash ? 'new' : 'old' ) - ); - ++$index; - $duplicates[ $item['item_url'] ] = $item['item_title']; - continue; - } - $author = ''; if ( $item['item_author'] ) { if ( is_string( $item['item_author'] ) ) { @@ -2777,9 +2845,22 @@ function ( $term ) { // we can use this to associate the items that were imported in a particular run. update_post_meta( $new_post_id, 'feedzy_job_time', $last_run ); + // Persist the successful item before handing control to third-party + // callbacks, so a throwing callback cannot cause a re-import. + if ( $use_new_hash ) { + update_post_meta( $job->ID, 'imported_items_hash', $imported_items ); + } else { + update_post_meta( $job->ID, 'imported_items', $imported_items ); + } + update_post_meta( $job->ID, 'imported_items_count', $count ); + do_action( 'feedzy_after_post_import', $new_post_id, $item, $this->settings ); } + if ( ! $batch_stopped ) { + delete_post_meta( $job->ID, 'import_batch_cursor' ); + } + if ( $use_new_hash ) { update_post_meta( $job->ID, 'imported_items_hash', $imported_items ); } else { @@ -3156,8 +3237,7 @@ private function try_save_featured_image( $img_source_url, $post_id, $post_title ) ); - // phpcs:ignore WordPressVIPMinimum.Functions.RestrictedFunctions.file_ops_unlink - unlink( $local_file ); + wp_delete_file( $local_file ); return false; } @@ -3174,8 +3254,7 @@ private function try_save_featured_image( $img_source_url, $post_id, $post_title ) ); - // phpcs:ignore WordPressVIPMinimum.Functions.RestrictedFunctions.file_ops_unlink - unlink( $local_file ); + wp_delete_file( $local_file ); return false; } @@ -3201,8 +3280,7 @@ private function try_save_featured_image( $img_source_url, $post_id, $post_title ) ); - // phpcs:ignore WordPressVIPMinimum.Functions.RestrictedFunctions.file_ops_unlink - unlink( $local_file ); + wp_delete_file( $local_file ); return false; } @@ -3313,8 +3391,7 @@ private function try_save_featured_image( $img_source_url, $post_id, $post_title ) ); - // phpcs:ignore WordPressVIPMinimum.Functions.RestrictedFunctions.file_ops_unlink - unlink( $local_file ); + wp_delete_file( $local_file ); return false; } @@ -3334,10 +3411,8 @@ private function try_save_featured_image( $img_source_url, $post_id, $post_title ) ); - // phpcs:ignore WordPressVIPMinimum.Functions.RestrictedFunctions.file_ops_unlink if ( file_exists( $file_array['tmp_name'] ) ) { - // phpcs:ignore WordPressVIPMinimum.Functions.RestrictedFunctions.file_ops_unlink - unlink( $file_array['tmp_name'] ); + wp_delete_file( $file_array['tmp_name'] ); } return false; @@ -3395,7 +3470,7 @@ public function add_cron() { if ( ( isset( $_POST['nonce'] ) && isset( $_POST['tab'] ) ) && - wp_verify_nonce( sanitize_text_field( wp_unslash( $_POST['nonce'] ) ), sanitize_text_field( $_POST['tab'] ) ) + wp_verify_nonce( sanitize_text_field( wp_unslash( $_POST['nonce'] ) ), sanitize_text_field( wp_unslash( $_POST['tab'] ) ) ) ) { if ( ! empty( $_POST['fz_cron_schedule'] ) ) { $schedule = sanitize_text_field( wp_unslash( $_POST['fz_cron_schedule'] ) ); @@ -3649,7 +3724,7 @@ public function integration_tabs( $tabs ) { public function save_tab_settings( $settings, $tab ) { if ( ! isset( $_POST['nonce'] ) || - ! wp_verify_nonce( sanitize_text_field( $_POST['nonce'] ), $tab ) + ! wp_verify_nonce( sanitize_text_field( wp_unslash( $_POST['nonce'] ) ), $tab ) ) { return array(); } @@ -4018,6 +4093,7 @@ private function purge_data() { } delete_post_meta( $id, 'imported_items_hash' ); + delete_post_meta( (int) $id, 'import_batch_cursor' ); delete_post_meta( $id, 'last_run' ); wp_die(); } @@ -4285,6 +4361,7 @@ public function feedzy_clone_import_job() { 'last_run', 'imported_items_hash', 'imported_items_count', + 'import_batch_cursor', ); if ( $post_meta ) { diff --git a/includes/views/import-metabox-edit.php b/includes/views/import-metabox-edit.php index 67555ef7..cfedb03e 100644 --- a/includes/views/import-metabox-edit.php +++ b/includes/views/import-metabox-edit.php @@ -6,7 +6,13 @@ * @package feedzy-rss-feeds-pro */ +if ( ! defined( 'ABSPATH' ) ) { + exit; // Exit if accessed directly. +} + global $post; +$import_batch_size = $import_batch_size ?? ''; +$import_item_delay_ms = $import_item_delay_ms ?? ''; ?>
@@ -620,10 +626,10 @@ class="fz-switch-toggle" type="checkbox" value="yes"
- +
- ', '' ) ); ?> +
@@ -828,6 +834,31 @@ class="feedzy-remove-media btn btn-outline-primary
+
+ +
+

PRO' : ''; ?>

+
+
+
+
+
+ + +
+
+
+
+
+ + +
+
+
+
+
+
+

diff --git a/readme.md b/readme.md index 6b4654c7..f33b0d0b 100644 --- a/readme.md +++ b/readme.md @@ -63,6 +63,7 @@ Take your “Feed to Post” automation to the next level with powerful Pro-only * **Auto-Delete Old Posts:** Remove outdated or time-sensitive imports automatically after a set number of days. * **Fallback Image:** Assign a default image for feed items without images to maintain a consistent site layout. * **Import Item Count Control:** Decide how many feed items to fetch from each source every run. +* **Import Batching:** Limit how many new items are processed in one cron request and optionally add a delay between items. On memory-restricted hosts, start with a batch size of 1–3 under General feed settings → Advanced. Developers can override these values with the `feedzy_import_batch_size` and `feedzy_import_item_delay_ms` filters. * **Scheduled Imports:** Run imports automatically on your preferred schedule with default options like hourly, daily, or weekly, or create custom cron schedules for complete flexibility. * **Custom Post Author:** Assign imported posts to a specific author profile. * **Post Excerpts and Custom Fields:** Map feed data to post excerpts or custom fields for richer content display. diff --git a/readme.txt b/readme.txt index 6046a9e2..de2be07c 100644 --- a/readme.txt +++ b/readme.txt @@ -63,6 +63,7 @@ Take your “Feed to Post” automation to the next level with powerful Pro-only * **Auto-Delete Old Posts:** Remove outdated or time-sensitive imports automatically after a set number of days. * **Fallback Image:** Assign a default image for feed items without images to maintain a consistent site layout. * **Import Item Count Control:** Decide how many feed items to fetch from each source every run. +* **Import Batching:** Limit how many new items are processed in one cron request and optionally add a delay between items. On memory-restricted hosts, start with a batch size of 1–3 under General feed settings → Advanced. Developers can override these values with the `feedzy_import_batch_size` and `feedzy_import_item_delay_ms` filters. * **Scheduled Imports:** Run imports automatically on your preferred schedule with default options like hourly, daily, or weekly, or create custom cron schedules for complete flexibility. * **Custom Post Author:** Assign imported posts to a specific author profile. * **Post Excerpts and Custom Fields:** Map feed data to post excerpts or custom fields for richer content display. diff --git a/tests/test-import.php b/tests/test-import.php index fbbc4515..e40e9ca3 100644 --- a/tests/test-import.php +++ b/tests/test-import.php @@ -89,7 +89,9 @@ public function test_feedzy_imports( $random_name1, $random_name2, $urls, $magic $_POST['feedzy_meta_data']['import_post_date'] = '[#item_date]'; $_POST['feedzy_meta_data']['import_post_content'] = "{$magic_tags}"; $_POST['feedzy_meta_data']['import_post_featured_img'] = '[#item_image]'; - $_POST['feedzy_meta_data']['import_feed_limit'] = $num_items; + $_POST['feedzy_meta_data']['import_feed_limit'] = $num_items; + $_POST['feedzy_meta_data']['import_batch_size'] = 1; + $_POST['feedzy_meta_data']['import_item_delay_ms'] = 100; $_POST['custom_vars_key'] = array(); $_POST['custom_vars_value'] = array(); @@ -98,6 +100,7 @@ public function test_feedzy_imports( $random_name1, $random_name2, $urls, $magic $_POST['feedzy_meta_data']['mark_duplicate_tag'] = '[#item_author]'; } + update_post_meta( $p->ID, 'import_batch_cursor', 'stale-cursor' ); do_action( 'save_post_feedzy_imports', $p->ID, $p ); $this->assertEquals( $p->post_title, $random_name2 ); $this->assertEquals( $p->post_type, 'feedzy_imports' ); @@ -114,6 +117,8 @@ public function test_feedzy_imports( $random_name1, $random_name2, $urls, $magic $import_featured_img = get_post_meta( $p->ID, 'import_post_featured_img', true ); $import_custom_fields = get_post_meta( $p->ID, 'imports_custom_fields', true ); $import_feed_limit = get_post_meta( $p->ID, 'import_feed_limit', true ); + $import_batch_size = get_post_meta( $p->ID, 'import_batch_size', true ); + $import_item_delay_ms = get_post_meta( $p->ID, 'import_item_delay_ms', true ); // The import_post_content goes through escape_html_to_tag() which converts HTML tags to JSON format $expected_content = escape_html_to_tag( $magic_tags ); @@ -135,6 +140,9 @@ public function test_feedzy_imports( $random_name1, $random_name2, $urls, $magic $this->assertEquals( '[#item_image]', $import_featured_img ); $this->assertEquals( '', $import_custom_fields ); $this->assertEquals( $num_items, $import_feed_limit ); + $this->assertEquals( 1, $import_batch_size ); + $this->assertEquals( 100, $import_item_delay_ms ); + $this->assertSame( '', get_post_meta( $p->ID, 'import_batch_cursor', true ) ); if ( $check_duplicate ) { $this->assertEquals( 'yes', $remove_duplicates );