diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index ecbd01aac15c43..58187b7dda4821 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -1205,9 +1205,12 @@ void report_tablet_callback(StorageEngine& engine, const ClusterInfo* cluster_in engine.tablet_manager()->get_partitions_visible_version(&partitions_version); request.__set_partitions_version(std::move(partitions_version)); + auto* metrics = DorisMetrics::instance(); int64_t max_compaction_score = - std::max(DorisMetrics::instance()->tablet_cumulative_max_compaction_score->value(), - DorisMetrics::instance()->tablet_base_max_compaction_score->value()); + std::max(metrics->tablet_cumulative_max_compaction_score->value(), + metrics->tablet_base_max_compaction_score->value()); + max_compaction_score = std::max(max_compaction_score, + metrics->tablet_time_series_max_compaction_score->value()); request.__set_tablet_max_compaction_score(max_compaction_score); request.__set_report_version(report_version); diff --git a/be/src/cloud/cloud_storage_engine.cpp b/be/src/cloud/cloud_storage_engine.cpp index 169486bf2563ee..9646beb6da9288 100644 --- a/be/src/cloud/cloud_storage_engine.cpp +++ b/be/src/cloud/cloud_storage_engine.cpp @@ -645,9 +645,12 @@ bool CloudStorageEngine::register_index_change_compaction( std::vector CloudStorageEngine::_generate_cloud_compaction_tasks( CompactionType compaction_type, bool check_score) { + DCHECK(compaction_type == CompactionType::BASE_COMPACTION || + compaction_type == CompactionType::CUMULATIVE_COMPACTION); std::vector> tablets_compaction; - int64_t max_compaction_score = 0; + CompactionScoreStats score_stats; + bool got_score_stats = false; std::unordered_set tablet_preparing_cumu_compaction; std::unordered_map>> submitted_cumu_compactions; @@ -724,22 +727,29 @@ std::vector CloudStorageEngine::_generate_cloud_compaction_task do { std::vector tablets; auto st = tablet_mgr().get_topn_tablets_to_compact(n, compaction_type, filter_out, &tablets, - &max_compaction_score); + &score_stats); if (!st.ok()) { LOG(WARNING) << "failed to get tablets to compact, err=" << st; break; } + got_score_stats = true; if (!need_pick_tablet) break; tablets_compaction = std::move(tablets); } while (false); - if (max_compaction_score > 0) { - if (compaction_type == CompactionType::BASE_COMPACTION) { + if (got_score_stats && score_stats.scanned) { + if (compaction_type == CompactionType::BASE_COMPACTION && score_stats.max_score > 0) { DorisMetrics::instance()->tablet_base_max_compaction_score->set_value( - max_compaction_score); - } else { - DorisMetrics::instance()->tablet_cumulative_max_compaction_score->set_value( - max_compaction_score); + score_stats.max_score); + } else if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { + if (check_score || score_stats.size_based_max_score > 0) { + DorisMetrics::instance()->tablet_cumulative_max_compaction_score->set_value( + score_stats.size_based_max_score); + } + if (check_score || score_stats.time_series_max_score > 0) { + DorisMetrics::instance()->tablet_time_series_max_compaction_score->set_value( + score_stats.time_series_max_score); + } } } diff --git a/be/src/cloud/cloud_storage_engine.h b/be/src/cloud/cloud_storage_engine.h index a737b1b0a3e010..efe42f1b227393 100644 --- a/be/src/cloud/cloud_storage_engine.h +++ b/be/src/cloud/cloud_storage_engine.h @@ -193,6 +193,11 @@ class CloudStorageEngine final : public BaseStorageEngine { void set_startup_timepoint(const std::chrono::time_point& tp) { _startup_timepoint = tp; } + + std::vector generate_cloud_compaction_tasks_for_test( + CompactionType compaction_type, bool check_score) { + return _generate_cloud_compaction_tasks(compaction_type, check_score); + } #endif private: diff --git a/be/src/cloud/cloud_tablet_mgr.cpp b/be/src/cloud/cloud_tablet_mgr.cpp index 3e979864138645..86c5407fb16ca4 100644 --- a/be/src/cloud/cloud_tablet_mgr.cpp +++ b/be/src/cloud/cloud_tablet_mgr.cpp @@ -29,6 +29,7 @@ #include "common/status.h" #include "cpp/sync_point.h" #include "runtime/memory/cache_policy.h" +#include "storage/compaction/cumulative_compaction_time_series_policy.h" #include "util/debug_points.h" #include "util/lru_cache.h" #include "util/stack_util.h" @@ -426,10 +427,11 @@ void CloudTabletMgr::sync_tablets(const CountDownLatch& stop_latch) { Status CloudTabletMgr::get_topn_tablets_to_compact( int n, CompactionType compaction_type, const std::function& filter_out, - std::vector>* tablets, int64_t* max_score) { + std::vector>* tablets, CompactionScoreStats* score_stats) { DCHECK(compaction_type == CompactionType::BASE_COMPACTION || compaction_type == CompactionType::CUMULATIVE_COMPACTION); - *max_score = 0; + *score_stats = {}; + score_stats->scanned = true; int64_t max_score_tablet_id = 0; // clang-format off auto score = [compaction_type](CloudTablet* t) { @@ -489,9 +491,18 @@ Status CloudTabletMgr::get_topn_tablets_to_compact( int64_t s = score(t.get()); if (s <= 0) { continue; } - if (s > *max_score) { + if (s > score_stats->max_score) { max_score_tablet_id = t->tablet_id(); - *max_score = s; + score_stats->max_score = s; + } + if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { + int64_t* policy_max_score = + t->tablet_meta()->compaction_policy() == CUMULATIVE_TIME_SERIES_POLICY + ? &score_stats->time_series_max_score + : &score_stats->size_based_max_score; + if (s > *policy_max_score) { + *policy_max_score = s; + } } if (filter_out(t.get())) { ++num_filtered; continue; } @@ -506,7 +517,7 @@ Status CloudTabletMgr::get_topn_tablets_to_compact( LOG_EVERY_N(INFO, 1000) << "get_topn_compaction_score, n=" << n << " type=" << compaction_type << " num_tablets=" << weak_tablets.size() << " num_skipped=" << num_skipped << " num_disabled=" << num_disabled << " num_filtered=" << num_filtered - << " max_score=" << *max_score << " max_score_tablet=" << max_score_tablet_id + << " max_score=" << score_stats->max_score << " max_score_tablet=" << max_score_tablet_id << " tablets=[" << [&buf] { std::stringstream ss; for (auto& i : buf) ss << i.first->tablet_id() << ":" << i.second << ","; return ss.str(); }() << "]" ; // clang-format on diff --git a/be/src/cloud/cloud_tablet_mgr.h b/be/src/cloud/cloud_tablet_mgr.h index 9894d97552b872..c44f4f36f8c3e6 100644 --- a/be/src/cloud/cloud_tablet_mgr.h +++ b/be/src/cloud/cloud_tablet_mgr.h @@ -79,13 +79,13 @@ class CloudTabletMgr { * @param filter_out a filter takes a tablet and return bool to check * whether skipping the tablet, true for skip * @param tablets output param - * @param max_score output param, max score of existed tablets + * @param score_stats output param, max scores of existed tablets * @return status of this call */ Status get_topn_tablets_to_compact(int n, CompactionType compaction_type, const std::function& filter_out, std::vector>* tablets, - int64_t* max_score); + CompactionScoreStats* score_stats); /** * Gets tablets info and total tablet num that are reported diff --git a/be/src/common/metrics/doris_metrics.cpp b/be/src/common/metrics/doris_metrics.cpp index 0c8716ec1f65ae..6f2df4150cf517 100644 --- a/be/src/common/metrics/doris_metrics.cpp +++ b/be/src/common/metrics/doris_metrics.cpp @@ -185,6 +185,7 @@ DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(process_fd_num_limit_soft, MetricUnit::NOUNIT DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(process_fd_num_limit_hard, MetricUnit::NOUNIT); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(tablet_cumulative_max_compaction_score, MetricUnit::NOUNIT); +DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(tablet_time_series_max_compaction_score, MetricUnit::NOUNIT); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(tablet_base_max_compaction_score, MetricUnit::NOUNIT); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(all_rowsets_num, MetricUnit::NOUNIT); @@ -376,6 +377,7 @@ DorisMetrics::DorisMetrics() : _metric_registry(_s_registry_name) { INT_GAUGE_METRIC_REGISTER(_server_metric_entity, process_fd_num_limit_hard); INT_GAUGE_METRIC_REGISTER(_server_metric_entity, tablet_cumulative_max_compaction_score); + INT_GAUGE_METRIC_REGISTER(_server_metric_entity, tablet_time_series_max_compaction_score); INT_GAUGE_METRIC_REGISTER(_server_metric_entity, tablet_base_max_compaction_score); INT_GAUGE_METRIC_REGISTER(_server_metric_entity, all_rowsets_num); diff --git a/be/src/common/metrics/doris_metrics.h b/be/src/common/metrics/doris_metrics.h index a91fb92feed94b..8aaac01834097b 100644 --- a/be/src/common/metrics/doris_metrics.h +++ b/be/src/common/metrics/doris_metrics.h @@ -155,9 +155,10 @@ class DorisMetrics { IntGauge* process_fd_num_limit_hard = nullptr; // the max compaction score of all tablets. - // Record base and cumulative scores separately, because - // we need to get the larger of the two. + // Record base, size-based cumulative and time-series cumulative scores separately, + // because we need to get the larger of them. IntGauge* tablet_cumulative_max_compaction_score = nullptr; + IntGauge* tablet_time_series_max_compaction_score = nullptr; IntGauge* tablet_base_max_compaction_score = nullptr; IntGauge* all_rowsets_num = nullptr; diff --git a/be/src/storage/data_dir.h b/be/src/storage/data_dir.h index 4598f3d87719fe..f5a2c57f33f8af 100644 --- a/be/src/storage/data_dir.h +++ b/be/src/storage/data_dir.h @@ -141,6 +141,13 @@ class DataDir { (double)_disk_capacity_bytes; } +#ifdef BE_TEST + void set_capacity_for_test(size_t disk_capacity_bytes, size_t available_bytes) { + _disk_capacity_bytes = disk_capacity_bytes; + _available_bytes = available_bytes; + } +#endif + // Move tablet to trash. Status move_to_trash(const std::string& tablet_path); diff --git a/be/src/storage/olap_common.h b/be/src/storage/olap_common.h index 1fb680ee747cc7..c2c439c19dd0ae 100644 --- a/be/src/storage/olap_common.h +++ b/be/src/storage/olap_common.h @@ -58,6 +58,13 @@ using TabletUid = UniqueId; enum CompactionType { BASE_COMPACTION = 1, CUMULATIVE_COMPACTION = 2, FULL_COMPACTION = 3 }; +struct CompactionScoreStats { + int64_t max_score = 0; + int64_t size_based_max_score = 0; + int64_t time_series_max_score = 0; + bool scanned = false; +}; + enum DataDirType { SPILL_DISK_DIR, OLAP_DATA_DIR, diff --git a/be/src/storage/olap_server.cpp b/be/src/storage/olap_server.cpp index 39cf4e081cec71..0b3ec3ec9de742 100644 --- a/be/src/storage/olap_server.cpp +++ b/be/src/storage/olap_server.cpp @@ -150,11 +150,12 @@ bool CompactionSubmitRegistry::has_compaction_task(DataDir* dir, CompactionType std::vector CompactionSubmitRegistry::pick_topn_tablets_for_compaction( TabletManager* tablet_mgr, DataDir* data_dir, CompactionType compaction_type, - const CumuCompactionPolicyTable& cumu_compaction_policies, uint32_t* disk_max_score) { + const CumuCompactionPolicyTable& cumu_compaction_policies, + CompactionScoreStats* disk_score_stats) { // non-lock, used in snapshot return tablet_mgr->find_best_tablets_to_compaction(compaction_type, data_dir, _get_tablet_set(data_dir, compaction_type), - disk_max_score, cumu_compaction_policies); + disk_score_stats, cumu_compaction_policies); } bool CompactionSubmitRegistry::insert(TabletSharedPtr tablet, CompactionType compaction_type) { @@ -937,7 +938,7 @@ bool need_generate_compaction_tasks(int task_cnt_per_disk, int thread_per_disk, return true; } -int get_concurrent_per_disk(int max_score, int thread_per_disk) { +int get_concurrent_per_disk(int64_t max_score, int thread_per_disk) { if (!config::enable_compaction_priority_scheduling) { return thread_per_disk; } @@ -976,11 +977,14 @@ bool has_free_compaction_slot(CompactionSubmitRegistry* registry, DataDir* dir, std::vector StorageEngine::_generate_compaction_tasks( CompactionType compaction_type, std::vector& data_dirs, bool check_score) { + DCHECK(compaction_type == CompactionType::BASE_COMPACTION || + compaction_type == CompactionType::CUMULATIVE_COMPACTION); TEST_SYNC_POINT_RETURN_WITH_VALUE("olap_server::_generate_compaction_tasks.return_empty", std::vector {}); _update_cumulative_compaction_policy(); std::vector tablets_compaction; - uint32_t max_compaction_score = 0; + CompactionScoreStats max_score_stats; + bool skipped_capacity_limited_dir = false; std::random_device rd; std::mt19937 g(rd()); @@ -1001,35 +1005,55 @@ std::vector StorageEngine::_generate_compaction_tasks( // Even if need_pick_tablet is false, we still need to call find_best_tablet_to_compaction(), // So that we can update the max_compaction_score metric. - if (!data_dir->reach_capacity_limit(0)) { - uint32_t disk_max_score = 0; - auto tablets = compaction_registry_snapshot.pick_topn_tablets_for_compaction( - _tablet_manager.get(), data_dir, compaction_type, - _cumulative_compaction_policies, &disk_max_score); - int concurrent_num = - get_concurrent_per_disk(disk_max_score, disk_compaction_slot_num(*data_dir)); - need_pick_tablet = need_generate_compaction_tasks( - executing_task_num, concurrent_num, compaction_type, - !compaction_registry_snapshot.has_compaction_task( - data_dir, CompactionType::CUMULATIVE_COMPACTION)); - for (const auto& tablet : tablets) { - if (tablet != nullptr) { - if (need_pick_tablet) { - tablets_compaction.emplace_back(tablet); - } - max_compaction_score = std::max(max_compaction_score, disk_max_score); + if (data_dir->reach_capacity_limit(0)) { + skipped_capacity_limited_dir = true; + continue; + } + + CompactionScoreStats disk_score_stats; + auto tablets = compaction_registry_snapshot.pick_topn_tablets_for_compaction( + _tablet_manager.get(), data_dir, compaction_type, _cumulative_compaction_policies, + &disk_score_stats); + max_score_stats.scanned = max_score_stats.scanned || disk_score_stats.scanned; + max_score_stats.max_score = std::max(max_score_stats.max_score, disk_score_stats.max_score); + max_score_stats.size_based_max_score = std::max(max_score_stats.size_based_max_score, + disk_score_stats.size_based_max_score); + max_score_stats.time_series_max_score = std::max(max_score_stats.time_series_max_score, + disk_score_stats.time_series_max_score); + int concurrent_num = get_concurrent_per_disk(disk_score_stats.max_score, + disk_compaction_slot_num(*data_dir)); + need_pick_tablet = need_generate_compaction_tasks( + executing_task_num, concurrent_num, compaction_type, + !compaction_registry_snapshot.has_compaction_task( + data_dir, CompactionType::CUMULATIVE_COMPACTION)); + for (const auto& tablet : tablets) { + if (tablet != nullptr) { + if (need_pick_tablet) { + tablets_compaction.emplace_back(tablet); } } } } - if (max_compaction_score > 0) { - if (compaction_type == CompactionType::BASE_COMPACTION) { + if (max_score_stats.scanned) { + if (compaction_type == CompactionType::BASE_COMPACTION && max_score_stats.max_score > 0) { DorisMetrics::instance()->tablet_base_max_compaction_score->set_value( - max_compaction_score); - } else { - DorisMetrics::instance()->tablet_cumulative_max_compaction_score->set_value( - max_compaction_score); + max_score_stats.max_score); + } else if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { + auto update_policy_score = [skipped_capacity_limited_dir, check_score](IntGauge* metric, + int64_t score) { + if (skipped_capacity_limited_dir) { + if (score > metric->value()) { + metric->set_value(score); + } + } else if (check_score || score > 0) { + metric->set_value(score); + } + }; + update_policy_score(DorisMetrics::instance()->tablet_cumulative_max_compaction_score, + max_score_stats.size_based_max_score); + update_policy_score(DorisMetrics::instance()->tablet_time_series_max_compaction_score, + max_score_stats.time_series_max_score); } } return tablets_compaction; diff --git a/be/src/storage/storage_engine.h b/be/src/storage/storage_engine.h index 0ccc851bc22cad..295f44e4e6241b 100644 --- a/be/src/storage/storage_engine.h +++ b/be/src/storage/storage_engine.h @@ -233,7 +233,8 @@ class CompactionSubmitRegistry { std::vector pick_topn_tablets_for_compaction( TabletManager* tablet_mgr, DataDir* data_dir, CompactionType compaction_type, - const CumuCompactionPolicyTable& cumu_compaction_policies, uint32_t* disk_max_score); + const CumuCompactionPolicyTable& cumu_compaction_policies, + CompactionScoreStats* disk_score_stats); private: TabletSet& _get_tablet_set(DataDir* dir, CompactionType compaction_type); @@ -381,6 +382,17 @@ class StorageEngine final : public BaseStorageEngine { int64_t get_compaction_num_per_round() const { return _compaction_num_per_round; } +#ifdef BE_TEST + std::vector generate_compaction_tasks_for_test( + CompactionType compaction_type, std::vector& data_dirs, bool check_score) { + return _generate_compaction_tasks(compaction_type, data_dirs, check_score); + } + + CompactionSubmitRegistry& compaction_submit_registry_for_test() { + return _compaction_submit_registry; + } +#endif + private: // Instance should be inited from `static open()` // MUST NOT be called in other circumstances. diff --git a/be/src/storage/tablet/tablet_manager.cpp b/be/src/storage/tablet/tablet_manager.cpp index 4d776827ae32d2..173c742a14f90f 100644 --- a/be/src/storage/tablet/tablet_manager.cpp +++ b/be/src/storage/tablet/tablet_manager.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -43,6 +44,7 @@ #include "io/fs/local_file_system.h" #include "runtime/exec_env.h" #include "service/backend_options.h" +#include "storage/compaction/cumulative_compaction_policy.h" #include "storage/compaction/cumulative_compaction_time_series_policy.h" #include "storage/data_dir.h" #include "storage/olap_common.h" @@ -741,9 +743,13 @@ struct TabletScore { std::vector TabletManager::find_best_tablets_to_compaction( CompactionType compaction_type, DataDir* data_dir, - const std::unordered_set& tablet_submitted_compaction, uint32_t* score, + const std::unordered_set& tablet_submitted_compaction, + CompactionScoreStats* score_stats, const std::unordered_map>& all_cumulative_compaction_policies) { + DCHECK(score_stats != nullptr); + *score_stats = {}; + score_stats->scanned = true; int64_t now_ms = UnixMillis(); const string& compaction_type_str = compaction_type == CompactionType::BASE_COMPACTION ? "base" : "cumulative"; @@ -805,8 +811,9 @@ std::vector TabletManager::find_best_tablets_to_compaction( return; } } - auto cumulative_compaction_policy = all_cumulative_compaction_policies.at( - tablet_ptr->tablet_meta()->compaction_policy()); + const auto& compaction_policy = tablet_ptr->tablet_meta()->compaction_policy(); + auto cumulative_compaction_policy = + all_cumulative_compaction_policies.at(compaction_policy); uint32_t current_compaction_score = tablet_ptr->calc_compaction_score(); if (current_compaction_score < 5) { tablet_ptr->set_skip_compaction(true, compaction_type, UnixSeconds()); @@ -816,12 +823,28 @@ std::vector TabletManager::find_best_tablets_to_compaction( return; } + std::optional suitable_for_compaction; + auto is_suitable = [&]() { + if (!suitable_for_compaction.has_value()) { + suitable_for_compaction = tablet_ptr->suitable_for_compaction( + compaction_type, cumulative_compaction_policy); + } + return suitable_for_compaction.value(); + }; + + if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { + int64_t* policy_max_score = compaction_policy == CUMULATIVE_TIME_SERIES_POLICY + ? &score_stats->time_series_max_score + : &score_stats->size_based_max_score; + if (current_compaction_score > *policy_max_score && is_suitable()) { + *policy_max_score = current_compaction_score; + } + } + // tablet should do single compaction if (current_compaction_score > single_compact_highest_score && tablet_ptr->should_fetch_from_peer()) { - bool ret = tablet_ptr->suitable_for_compaction(compaction_type, - cumulative_compaction_policy); - if (ret) { + if (is_suitable()) { single_compact_highest_score = current_compaction_score; best_single_compact_tablet = tablet_ptr; } @@ -834,9 +857,7 @@ std::vector TabletManager::find_best_tablets_to_compaction( if ((top_tablets.size() >= compaction_num_per_round && current_compaction_score > top_tablets.top().score) || top_tablets.size() < compaction_num_per_round) { - bool ret = tablet_ptr->suitable_for_compaction(compaction_type, - cumulative_compaction_policy); - if (ret) { + if (is_suitable()) { top_tablets.push(ts); if (top_tablets.size() > compaction_num_per_round) { top_tablets.pop(); @@ -846,9 +867,7 @@ std::vector TabletManager::find_best_tablets_to_compaction( } } else { if (current_compaction_score > highest_score && !tablet_ptr->should_fetch_from_peer()) { - bool ret = tablet_ptr->suitable_for_compaction(compaction_type, - cumulative_compaction_policy); - if (ret) { + if (is_suitable()) { highest_score = current_compaction_score; best_tablet = tablet_ptr; } @@ -887,8 +906,9 @@ std::vector TabletManager::find_best_tablets_to_compaction( << best_single_compact_tablet->should_fetch_from_peer(); picked_tablet.emplace_back(std::move(best_single_compact_tablet)); } - *score = highest_score > single_compact_highest_score ? highest_score - : single_compact_highest_score; + score_stats->max_score = highest_score > single_compact_highest_score + ? highest_score + : single_compact_highest_score; return picked_tablet; } diff --git a/be/src/storage/tablet/tablet_manager.h b/be/src/storage/tablet/tablet_manager.h index dccf5aaffad567..3cae490c87769c 100644 --- a/be/src/storage/tablet/tablet_manager.h +++ b/be/src/storage/tablet/tablet_manager.h @@ -82,7 +82,8 @@ class TabletManager { // single compaction tasks for the tablet. std::vector find_best_tablets_to_compaction( CompactionType compaction_type, DataDir* data_dir, - const std::unordered_set& tablet_submitted_compaction, uint32_t* score, + const std::unordered_set& tablet_submitted_compaction, + CompactionScoreStats* score_stats, const std::unordered_map>& all_cumulative_compaction_policies); diff --git a/be/test/cloud/cloud_compaction_test.cpp b/be/test/cloud/cloud_compaction_test.cpp index 05eec3149f996a..22d819b97a3e82 100644 --- a/be/test/cloud/cloud_compaction_test.cpp +++ b/be/test/cloud/cloud_compaction_test.cpp @@ -22,13 +22,16 @@ #include #include +#include #include "cloud/cloud_base_compaction.h" #include "cloud/cloud_cluster_info.h" #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet.h" #include "cloud/cloud_tablet_mgr.h" +#include "common/metrics/doris_metrics.h" #include "json2pb/json_to_pb.h" +#include "storage/compaction/cumulative_compaction_time_series_policy.h" #include "storage/olap_common.h" #include "storage/rowset/rowset_factory.h" #include "storage/rowset/rowset_meta.h" @@ -141,13 +144,16 @@ TEST_F(CloudCompactionTest, failure_base_compaction_tablet_sleep_test) { tablet1->set_last_base_compaction_failure_time(0); tablet1->tablet_meta()->tablet_schema()->set_disable_auto_compaction(false); tablet1->_approximate_num_rowsets = 10; + tablet1->_approximate_cumu_num_rowsets = 0; mgr.put_tablet_for_UT(tablet1); - int64_t max_score; + CompactionScoreStats score_stats; std::vector> tablets {}; Status st = mgr.get_topn_tablets_to_compact(1, CompactionType::BASE_COMPACTION, filter_out, - &tablets, &max_score); + &tablets, &score_stats); ASSERT_EQ(st, Status::OK()); + ASSERT_TRUE(score_stats.scanned); + ASSERT_EQ(score_stats.max_score, 10); ASSERT_EQ(tablets.size(), 1); tablet1->set_last_base_compaction_failure_time( @@ -155,8 +161,10 @@ TEST_F(CloudCompactionTest, failure_base_compaction_tablet_sleep_test) { std::chrono::system_clock::now().time_since_epoch()) .count()); st = mgr.get_topn_tablets_to_compact(1, CompactionType::BASE_COMPACTION, filter_out, &tablets, - &max_score); + &score_stats); ASSERT_EQ(st, Status::OK()); + ASSERT_TRUE(score_stats.scanned); + ASSERT_EQ(score_stats.max_score, 10); ASSERT_EQ(tablets.size(), 0); } @@ -182,11 +190,15 @@ TEST_F(CloudCompactionTest, failure_cumu_compaction_tablet_sleep_test) { tablet1->_approximate_cumu_num_deltas = 10; mgr.put_tablet_for_UT(tablet1); - int64_t max_score; + CompactionScoreStats score_stats; std::vector> tablets {}; Status st = mgr.get_topn_tablets_to_compact(1, CompactionType::CUMULATIVE_COMPACTION, - filter_out, &tablets, &max_score); + filter_out, &tablets, &score_stats); ASSERT_EQ(st, Status::OK()); + ASSERT_TRUE(score_stats.scanned); + ASSERT_EQ(score_stats.max_score, 10); + ASSERT_EQ(score_stats.size_based_max_score, 10); + ASSERT_EQ(score_stats.time_series_max_score, 0); ASSERT_EQ(tablets.size(), 1); tablet1->set_last_cumu_compaction_failure_time( @@ -194,11 +206,87 @@ TEST_F(CloudCompactionTest, failure_cumu_compaction_tablet_sleep_test) { std::chrono::system_clock::now().time_since_epoch()) .count()); st = mgr.get_topn_tablets_to_compact(1, CompactionType::BASE_COMPACTION, filter_out, &tablets, - &max_score); + &score_stats); ASSERT_EQ(st, Status::OK()); + ASSERT_TRUE(score_stats.scanned); + ASSERT_EQ(score_stats.max_score, 0); ASSERT_EQ(tablets.size(), 0); } +TEST_F(CloudCompactionTest, split_cumu_compaction_score_stats_before_filter) { + CloudTabletMgr mgr(_engine); + + auto create_tablet = [this, &mgr](int64_t tablet_id, std::string_view compaction_policy, + int64_t score) { + TabletMetaSharedPtr tablet_meta(new TabletMeta(*_tablet_meta)); + tablet_meta->_tablet_id = tablet_id; + tablet_meta->set_compaction_policy(std::string(compaction_policy)); + auto tablet = std::make_shared(_engine, tablet_meta); + tablet->tablet_meta()->tablet_schema()->set_disable_auto_compaction(false); + tablet->_approximate_cumu_num_deltas = score; + mgr.put_tablet_for_UT(tablet); + return tablet; + }; + + create_tablet(10000, CUMULATIVE_SIZE_BASED_POLICY, 7); + create_tablet(10001, CUMULATIVE_TIME_SERIES_POLICY, 13); + + auto filter_time_series = [](CloudTablet* t) { return t->tablet_id() == 10001; }; + CompactionScoreStats score_stats; + std::vector> tablets; + Status st = mgr.get_topn_tablets_to_compact(1, CompactionType::CUMULATIVE_COMPACTION, + filter_time_series, &tablets, &score_stats); + ASSERT_EQ(st, Status::OK()); + ASSERT_TRUE(score_stats.scanned); + ASSERT_EQ(score_stats.max_score, 13); + ASSERT_EQ(score_stats.size_based_max_score, 7); + ASSERT_EQ(score_stats.time_series_max_score, 13); + ASSERT_EQ(tablets.size(), 1); + ASSERT_EQ(tablets[0]->tablet_id(), 10000); +} + +TEST_F(CloudCompactionTest, generate_cloud_compaction_tasks_updates_policy_metrics) { + CloudTabletMgr& mgr = _engine.tablet_mgr(); + TabletMetaSharedPtr tablet_meta(new TabletMeta(*_tablet_meta)); + tablet_meta->_tablet_id = 11000; + tablet_meta->set_compaction_policy(std::string(CUMULATIVE_SIZE_BASED_POLICY)); + auto tablet = std::make_shared(_engine, tablet_meta); + tablet->tablet_meta()->tablet_schema()->set_disable_auto_compaction(false); + tablet->_approximate_cumu_num_deltas = 7; + mgr.put_tablet_for_UT(tablet); + + auto* metrics = DorisMetrics::instance(); + metrics->tablet_cumulative_max_compaction_score->set_value(101); + metrics->tablet_time_series_max_compaction_score->set_value(200); + + auto tablets = _engine.generate_cloud_compaction_tasks_for_test( + CompactionType::CUMULATIVE_COMPACTION, false); + ASSERT_EQ(tablets.size(), 1); + ASSERT_EQ(tablets[0]->tablet_id(), 11000); + ASSERT_EQ(metrics->tablet_cumulative_max_compaction_score->value(), 7); + ASSERT_EQ(metrics->tablet_time_series_max_compaction_score->value(), 200); + + tablets = _engine.generate_cloud_compaction_tasks_for_test( + CompactionType::CUMULATIVE_COMPACTION, true); + ASSERT_EQ(tablets.size(), 1); + ASSERT_EQ(tablets[0]->tablet_id(), 11000); + ASSERT_EQ(metrics->tablet_cumulative_max_compaction_score->value(), 7); + ASSERT_EQ(metrics->tablet_time_series_max_compaction_score->value(), 0); +} + +TEST_F(CloudCompactionTest, generate_cloud_compaction_tasks_clears_metrics_without_tablets) { + auto* metrics = DorisMetrics::instance(); + metrics->tablet_cumulative_max_compaction_score->set_value(101); + metrics->tablet_time_series_max_compaction_score->set_value(200); + + auto tablets = _engine.generate_cloud_compaction_tasks_for_test( + CompactionType::CUMULATIVE_COMPACTION, true); + + ASSERT_TRUE(tablets.empty()); + ASSERT_EQ(metrics->tablet_cumulative_max_compaction_score->value(), 0); + ASSERT_EQ(metrics->tablet_time_series_max_compaction_score->value(), 0); +} + static RowsetSharedPtr create_rowset(Version version, int num_segments, bool overlapping, int data_size) { auto rs_meta = std::make_shared(); diff --git a/be/test/storage/tablet/tablet_mgr_test.cpp b/be/test/storage/tablet/tablet_mgr_test.cpp index 3737742f014690..1a69859db323f9 100644 --- a/be/test/storage/tablet/tablet_mgr_test.cpp +++ b/be/test/storage/tablet/tablet_mgr_test.cpp @@ -26,9 +26,11 @@ #include #include #include +#include #include #include "common/config.h" +#include "common/metrics/doris_metrics.h" #include "common/status.h" #include "gtest/gtest_pred_impl.h" #include "io/fs/local_file_system.h" @@ -47,6 +49,8 @@ #include "storage/tablet/tablet_manager.h" #include "storage/tablet/tablet_meta.h" #include "storage/tablet/tablet_meta_manager.h" +#include "util/debug_points.h" +#include "util/defer_op.h" #include "util/uid_util.h" using ::testing::_; @@ -88,6 +92,91 @@ class TabletMgrTest : public testing::Test { _tablet_mgr = nullptr; config::compaction_num_per_round = 1; } + + TabletSharedPtr create_compaction_tablet( + int64_t tablet_id, bool enable_single_compact, int rowset_size, + std::string_view compaction_policy = CUMULATIVE_SIZE_BASED_POLICY, + DataDir* data_dir = nullptr) { + data_dir = data_dir == nullptr ? _data_dir : data_dir; + std::vector cols; + TColumn col1; + col1.column_type.type = TPrimitiveType::SMALLINT; + col1.__set_column_name("col1"); + col1.__set_is_key(true); + cols.push_back(col1); + + TColumn col2; + col2.column_type.type = TPrimitiveType::INT; + col2.__set_column_name(SEQUENCE_COL); + col2.__set_is_key(false); + col2.__set_aggregation_type(TAggregationType::REPLACE); + cols.push_back(col2); + + TColumn col3; + col3.column_type.type = TPrimitiveType::INT; + col3.__set_column_name("v1"); + col3.__set_is_key(false); + col3.__set_aggregation_type(TAggregationType::REPLACE); + cols.push_back(col3); + + RuntimeProfile profile("CreateTablet"); + TTabletSchema tablet_schema; + tablet_schema.__set_short_key_column_count(1); + tablet_schema.__set_schema_hash(3333); + tablet_schema.__set_keys_type(TKeysType::UNIQUE_KEYS); + tablet_schema.__set_storage_type(TStorageType::COLUMN); + tablet_schema.__set_columns(cols); + tablet_schema.__set_sequence_col_idx(1); + tablet_schema.__set_enable_single_replica_compaction(enable_single_compact); + TCreateTabletReq create_tablet_req; + create_tablet_req.__set_tablet_schema(tablet_schema); + create_tablet_req.__set_tablet_id(tablet_id); + create_tablet_req.__set_version(1); + create_tablet_req.__set_replica_id(tablet_id * 10); + create_tablet_req.__set_compaction_policy(std::string(compaction_policy)); + if (compaction_policy == CUMULATIVE_TIME_SERIES_POLICY) { + create_tablet_req.__set_time_series_compaction_file_count_threshold(1); + } + std::vector data_dirs; + data_dirs.push_back(data_dir); + Status create_st = _tablet_mgr->create_tablet(create_tablet_req, data_dirs, &profile); + if (!create_st.ok()) { + ADD_FAILURE() << create_st; + return nullptr; + } + + TabletSharedPtr tablet = _tablet_mgr->get_tablet(tablet_id); + if (tablet == nullptr) { + ADD_FAILURE() << "failed to get tablet " << tablet_id; + return nullptr; + } + + auto create_rowset = [=, this](int64_t start, int64_t end) { + auto rowset_meta = std::make_shared(); + Version version(start, end); + rowset_meta->set_version(version); + rowset_meta->set_tablet_id(tablet->tablet_id()); + rowset_meta->set_tablet_uid(tablet->tablet_uid()); + rowset_meta->set_rowset_id(k_engine->next_rowset_id()); + return std::make_shared(tablet->tablet_schema(), std::move(rowset_meta), + tablet->tablet_path()); + }; + auto st = tablet->init(); + if (!st.ok()) { + ADD_FAILURE() << st; + return nullptr; + } + for (int i = 2; i <= rowset_size; ++i) { + auto rs = create_rowset(i, i); + st = tablet->add_inc_rowset(rs); + if (!st.ok()) { + ADD_FAILURE() << st; + return nullptr; + } + } + return tablet; + } + StorageEngine* k_engine; private: @@ -345,7 +434,8 @@ TEST_F(TabletMgrTest, GetRowsetId) { } TEST_F(TabletMgrTest, FindTabletWithCompact) { - auto create_tablet = [this](int64_t tablet_id, bool enable_single_compact, int rowset_size) { + auto create_tablet = [this](int64_t tablet_id, bool enable_single_compact, int rowset_size, + std::string_view compaction_policy = CUMULATIVE_SIZE_BASED_POLICY) { std::vector cols; TColumn col1; col1.column_type.type = TPrimitiveType::SMALLINT; @@ -381,6 +471,10 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) { create_tablet_req.__set_tablet_id(tablet_id); create_tablet_req.__set_version(1); create_tablet_req.__set_replica_id(tablet_id * 10); + create_tablet_req.__set_compaction_policy(std::string(compaction_policy)); + if (compaction_policy == CUMULATIVE_TIME_SERIES_POLICY) { + create_tablet_req.__set_time_series_compaction_file_count_threshold(1); + } std::vector data_dirs; data_dirs.push_back(_data_dir); Status create_st = _tablet_mgr->create_tablet(create_tablet_req, data_dirs, &profile); @@ -434,13 +528,16 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) { cumulative_compaction_policies[CUMULATIVE_TIME_SERIES_POLICY] = CumulativeCompactionPolicyFactory::create_cumulative_compaction_policy( CUMULATIVE_TIME_SERIES_POLICY); - uint32_t score = 0; + CompactionScoreStats score_stats; auto compact_tablets = _tablet_mgr->find_best_tablets_to_compaction( - CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set, &score, + CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set, &score_stats, cumulative_compaction_policies); ASSERT_EQ(compact_tablets.size(), 1); ASSERT_EQ(compact_tablets[0]->tablet_id(), 10); - ASSERT_EQ(score, 14); + ASSERT_TRUE(score_stats.scanned); + ASSERT_EQ(score_stats.max_score, 14); + ASSERT_EQ(score_stats.size_based_max_score, 14); + ASSERT_EQ(score_stats.time_series_max_score, 0); // create 10 tablets enable single compact // 5 tablets do cumu compaction, 5 tablets do single compaction @@ -450,21 +547,25 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) { } compact_tablets = _tablet_mgr->find_best_tablets_to_compaction( - CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set, &score, + CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set, &score_stats, cumulative_compaction_policies); ASSERT_EQ(compact_tablets.size(), 2); ASSERT_EQ(compact_tablets[0]->tablet_id(), 19); ASSERT_EQ(compact_tablets[1]->tablet_id(), 20); - ASSERT_EQ(score, 24); + ASSERT_EQ(score_stats.max_score, 24); + ASSERT_EQ(score_stats.size_based_max_score, 24); + ASSERT_EQ(score_stats.time_series_max_score, 0); create_tablet(21, false, rowset_size++); compact_tablets = _tablet_mgr->find_best_tablets_to_compaction( - CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set, &score, + CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set, &score_stats, cumulative_compaction_policies); ASSERT_EQ(compact_tablets.size(), 1); ASSERT_EQ(compact_tablets[0]->tablet_id(), 21); - ASSERT_EQ(score, 25); + ASSERT_EQ(score_stats.max_score, 25); + ASSERT_EQ(score_stats.size_based_max_score, 25); + ASSERT_EQ(score_stats.time_series_max_score, 0); // drop all tablets for (int64_t id = 1; id <= 21; ++id) { @@ -472,6 +573,26 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) { ASSERT_TRUE(drop_st.ok()) << drop_st; } + { + create_tablet(40001, false, 8, CUMULATIVE_SIZE_BASED_POLICY); + create_tablet(40002, false, 12, CUMULATIVE_TIME_SERIES_POLICY); + + compact_tablets = _tablet_mgr->find_best_tablets_to_compaction( + CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set, &score_stats, + cumulative_compaction_policies); + ASSERT_TRUE(score_stats.scanned); + ASSERT_EQ(score_stats.max_score, 12); + ASSERT_EQ(score_stats.size_based_max_score, 8); + ASSERT_EQ(score_stats.time_series_max_score, 12); + ASSERT_EQ(compact_tablets.size(), 1); + ASSERT_EQ(compact_tablets[0]->tablet_id(), 40002); + + Status drop_st = _tablet_mgr->drop_tablet(40001, 400010, false); + ASSERT_TRUE(drop_st.ok()) << drop_st; + drop_st = _tablet_mgr->drop_tablet(40002, 400020, false); + ASSERT_TRUE(drop_st.ok()) << drop_st; + } + { k_engine->_compaction_num_per_round = 10; for (int64_t i = 1; i <= 100; ++i) { @@ -479,7 +600,7 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) { } compact_tablets = _tablet_mgr->find_best_tablets_to_compaction( - CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set, &score, + CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set, &score_stats, cumulative_compaction_policies); ASSERT_EQ(compact_tablets.size(), 10); int index = 0; @@ -504,7 +625,7 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) { create_tablet(20102, true, 200); compact_tablets = _tablet_mgr->find_best_tablets_to_compaction( - CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set, &score, + CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set, &score_stats, cumulative_compaction_policies); ASSERT_EQ(compact_tablets.size(), 11); for (int i = 0; i < 10; ++i) { @@ -532,7 +653,7 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) { } compact_tablets = _tablet_mgr->find_best_tablets_to_compaction( - CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set, &score, + CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set, &score_stats, cumulative_compaction_policies); ASSERT_EQ(compact_tablets.size(), 5); for (int i = 0; i < 5; ++i) { @@ -552,6 +673,157 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) { ASSERT_TRUE(trash_st.ok()) << trash_st; } +TEST_F(TabletMgrTest, FindBestTabletsIgnoresUnsuitablePolicyScore) { + auto tablet = create_compaction_tablet(50001, false, 12, CUMULATIVE_TIME_SERIES_POLICY); + ASSERT_TRUE(tablet != nullptr); + ASSERT_GT(tablet->calc_compaction_score(), 5); + + bool old_enable_debug_points = config::enable_debug_points; + config::enable_debug_points = true; + Defer restore_debug_points([&] { config::enable_debug_points = old_enable_debug_points; }); + DebugPoints::instance()->add("Tablet._calc_cumulative_compaction_score.return"); + Defer clear_debug_point([] { DebugPoints::instance()->clear(); }); + + std::unordered_set cumu_set; + std::unordered_map> + cumulative_compaction_policies; + cumulative_compaction_policies[CUMULATIVE_SIZE_BASED_POLICY] = + CumulativeCompactionPolicyFactory::create_cumulative_compaction_policy( + CUMULATIVE_SIZE_BASED_POLICY); + cumulative_compaction_policies[CUMULATIVE_TIME_SERIES_POLICY] = + CumulativeCompactionPolicyFactory::create_cumulative_compaction_policy( + CUMULATIVE_TIME_SERIES_POLICY); + + CompactionScoreStats score_stats; + auto compact_tablets = _tablet_mgr->find_best_tablets_to_compaction( + CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set, &score_stats, + cumulative_compaction_policies); + ASSERT_TRUE(score_stats.scanned); + ASSERT_EQ(score_stats.max_score, 0); + ASSERT_EQ(score_stats.size_based_max_score, 0); + ASSERT_EQ(score_stats.time_series_max_score, 0); + ASSERT_TRUE(compact_tablets.empty()); +} + +TEST_F(TabletMgrTest, GenerateCompactionTasksClearsMissingPolicyScoreOnCheck) { + auto tablet = create_compaction_tablet(51001, false, 8, CUMULATIVE_SIZE_BASED_POLICY); + ASSERT_TRUE(tablet != nullptr); + auto* metrics = DorisMetrics::instance(); + metrics->tablet_cumulative_max_compaction_score->set_value(101); + metrics->tablet_time_series_max_compaction_score->set_value(200); + + std::vector data_dirs {_data_dir}; + auto tasks = k_engine->generate_compaction_tasks_for_test(CompactionType::CUMULATIVE_COMPACTION, + data_dirs, true); + + ASSERT_EQ(tasks.size(), 1); + ASSERT_EQ(tasks[0]->tablet_id(), 51001); + ASSERT_EQ(metrics->tablet_cumulative_max_compaction_score->value(), 8); + ASSERT_EQ(metrics->tablet_time_series_max_compaction_score->value(), 0); +} + +TEST_F(TabletMgrTest, GenerateCompactionTasksKeepsMissingPolicyScoreWithoutCheck) { + auto tablet = create_compaction_tablet(52001, false, 8, CUMULATIVE_SIZE_BASED_POLICY); + ASSERT_TRUE(tablet != nullptr); + auto* metrics = DorisMetrics::instance(); + metrics->tablet_cumulative_max_compaction_score->set_value(101); + metrics->tablet_time_series_max_compaction_score->set_value(200); + + std::vector data_dirs {_data_dir}; + auto tasks = k_engine->generate_compaction_tasks_for_test(CompactionType::CUMULATIVE_COMPACTION, + data_dirs, false); + + ASSERT_EQ(tasks.size(), 1); + ASSERT_EQ(tasks[0]->tablet_id(), 52001); + ASSERT_EQ(metrics->tablet_cumulative_max_compaction_score->value(), 8); + ASSERT_EQ(metrics->tablet_time_series_max_compaction_score->value(), 200); +} + +TEST_F(TabletMgrTest, GenerateCompactionTasksDoesNotUpdateMetricWhenNoDirScanned) { + auto* metrics = DorisMetrics::instance(); + metrics->tablet_cumulative_max_compaction_score->set_value(101); + metrics->tablet_time_series_max_compaction_score->set_value(200); + + std::vector data_dirs; + auto tasks = k_engine->generate_compaction_tasks_for_test(CompactionType::CUMULATIVE_COMPACTION, + data_dirs, true); + + ASSERT_TRUE(tasks.empty()); + ASSERT_EQ(metrics->tablet_cumulative_max_compaction_score->value(), 101); + ASSERT_EQ(metrics->tablet_time_series_max_compaction_score->value(), 200); +} + +TEST_F(TabletMgrTest, GenerateCompactionTasksAggregatesScoreWhenNoSlot) { + auto dummy = create_compaction_tablet(53000, false, 5, CUMULATIVE_SIZE_BASED_POLICY); + auto size_based = create_compaction_tablet(53001, false, 8, CUMULATIVE_SIZE_BASED_POLICY); + auto time_series = create_compaction_tablet(53002, false, 12, CUMULATIVE_TIME_SERIES_POLICY); + ASSERT_TRUE(dummy != nullptr); + ASSERT_TRUE(size_based != nullptr); + ASSERT_TRUE(time_series != nullptr); + + std::vector data_dirs {_data_dir}; + auto& registry = k_engine->compaction_submit_registry_for_test(); + registry.reset(data_dirs); + Defer reset_registry([&] { registry.reset(data_dirs); }); + dummy->compaction_stage = CompactionStage::EXECUTING; + ASSERT_FALSE(registry.insert(dummy, CompactionType::CUMULATIVE_COMPACTION)); + + int32_t old_compaction_task_num_per_disk = config::compaction_task_num_per_disk; + config::compaction_task_num_per_disk = 1; + Defer restore_config( + [&] { config::compaction_task_num_per_disk = old_compaction_task_num_per_disk; }); + bool old_enable_compaction_priority_scheduling = config::enable_compaction_priority_scheduling; + config::enable_compaction_priority_scheduling = false; + Defer restore_priority_scheduling([&] { + config::enable_compaction_priority_scheduling = old_enable_compaction_priority_scheduling; + }); + + auto* metrics = DorisMetrics::instance(); + metrics->tablet_cumulative_max_compaction_score->set_value(0); + metrics->tablet_time_series_max_compaction_score->set_value(0); + + auto tasks = k_engine->generate_compaction_tasks_for_test(CompactionType::CUMULATIVE_COMPACTION, + data_dirs, true); + + ASSERT_TRUE(tasks.empty()); + ASSERT_EQ(metrics->tablet_cumulative_max_compaction_score->value(), 8); + ASSERT_EQ(metrics->tablet_time_series_max_compaction_score->value(), 12); +} + +TEST_F(TabletMgrTest, GenerateCompactionTasksDoesNotLowerPolicyScoreWhenDirFull) { + std::string full_dir_path = "./be/test/storage/test_data/converter_test_data/tmp_full"; + ASSERT_TRUE(io::global_local_filesystem()->delete_directory(full_dir_path).ok()); + ASSERT_TRUE(io::global_local_filesystem()->create_directory(full_dir_path).ok()); + ASSERT_TRUE(io::global_local_filesystem()->create_directory(full_dir_path + "/meta").ok()); + Defer cleanup_full_dir([&] { + static_cast(io::global_local_filesystem()->delete_directory(full_dir_path)); + }); + + auto full_data_dir = std::make_unique(*k_engine, full_dir_path, 1000000000); + ASSERT_TRUE(full_data_dir->init().ok()); + auto full_time_series = create_compaction_tablet( + 54001, false, 12, CUMULATIVE_TIME_SERIES_POLICY, full_data_dir.get()); + auto size_based = create_compaction_tablet(54002, false, 8, CUMULATIVE_SIZE_BASED_POLICY); + ASSERT_TRUE(full_time_series != nullptr); + ASSERT_TRUE(size_based != nullptr); + Defer drop_full_tablet( + [&] { static_cast(_tablet_mgr->drop_tablet(54001, 540010, false)); }); + full_data_dir->set_capacity_for_test(100, 0); + + auto* metrics = DorisMetrics::instance(); + metrics->tablet_cumulative_max_compaction_score->set_value(0); + metrics->tablet_time_series_max_compaction_score->set_value(200); + + std::vector data_dirs {_data_dir, full_data_dir.get()}; + auto tasks = k_engine->generate_compaction_tasks_for_test(CompactionType::CUMULATIVE_COMPACTION, + data_dirs, true); + + ASSERT_EQ(tasks.size(), 1); + ASSERT_EQ(tasks[0]->tablet_id(), 54002); + ASSERT_EQ(metrics->tablet_cumulative_max_compaction_score->value(), 8); + ASSERT_EQ(metrics->tablet_time_series_max_compaction_score->value(), 200); +} + TEST_F(TabletMgrTest, LoadTabletFromMeta) { TTabletId tablet_id = 111; TSchemaHash schema_hash = 3333; diff --git a/be/test/util/doris_metrics_test.cpp b/be/test/util/doris_metrics_test.cpp index 2efd0fad01072b..d55384897dd58e 100644 --- a/be/test/util/doris_metrics_test.cpp +++ b/be/test/util/doris_metrics_test.cpp @@ -157,6 +157,12 @@ TEST_F(DorisMetricsTest, Normal) { EXPECT_TRUE(metric != nullptr); EXPECT_STREQ("31", metric->to_string().c_str()); } + { + DorisMetrics::instance()->tablet_time_series_max_compaction_score->set_value(42); + auto metric = server_entity->get_metric("tablet_time_series_max_compaction_score"); + EXPECT_TRUE(metric != nullptr); + EXPECT_STREQ("42", metric->to_string().c_str()); + } { DorisMetrics::instance()->base_compaction_bytes_total->increment(32); auto metric =