Skip to content

Merge dev/refactor_create_seal_shard into stable/v4.x #449

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Hooper9973 merged 7 commits into from
Jul 7, 2026
Merged

Merge dev/refactor_create_seal_shard into stable/v4.x #449

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
46efb1e
SDSTOR-21981: make shard create/seal log-only, avoid header/footer I/…
Hooper9973 Jun 18, 2026
efba90e
Enhance unit tests with timeout handling for blob creation and synchr…
Hooper9973 Jun 24, 2026
6b3006b
Remove '-csv error' flag in CMakeLists.txt to get realtime log
Hooper9973 Jun 24, 2026
0644c35
Bump version to 4.2.1 and remove logging flags from CMake test config…
Hooper9973 Jun 24, 2026
340788e
SDSTOR-22421: shard superblk backward compat for log-only shard refac…
Hooper9973 Jun 29, 2026
3f38149
Remove unused shard header/footer generation during gc, and add valid…
Hooper9973 Jul 7, 2026
2939951
Merge remote-tracking branch 'upstream/dev/refactor_create_seal_shard…
Hooper9973 Jul 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .jenkins/Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@ pipeline {
conan create ${BUILD_MISSING} -s:h build_type=RelWithDebInfo -o sisl/*:malloc_impl=tcmalloc ${CONAN_FLAGS} . ; \
"
}
post {
failure { script { sleep 3600000 } }
}
}

stage("Deploy") {
Expand All @@ -82,7 +85,7 @@ pipeline {
expression { (env.BRANCH_NAME == "${TARGET_BRANCH}") }
expression { (!"${upstream_triggered}") || ("${upstream_triggered}" == "") }
} }

stages {
stage('StorageManager') {
steps {
Expand Down
4 changes: 2 additions & 2 deletions conanfile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

class HomeObjectConan(ConanFile):
name = "homeobject"
version = "4.1.24"
version = "4.2.3"

@JacksonYao287 JacksonYao287 Jul 7, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

4.2.1？

@Hooper9973 Hooper9973 Jul 7, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It doesn't matter~


homepage = "https://github.com/eBay/HomeObject"
description = "Blob Store built on HomeStore"
Expand All @@ -26,7 +26,7 @@ class HomeObjectConan(ConanFile):
"coverage": ['True', 'False'],
"sanitize": ['True', 'False'],
}

default_options = {
'shared': False,
'fPIC': True,
Expand Down
6 changes: 4 additions & 2 deletions src/include/homeobject/shard_manager.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,14 @@ struct ShardInfo {
shard_id_t id;
pg_id_t placement_group;
State state;
uint64_t lsn; // created_lsn
uint64_t create_lsn;
uint64_t created_time;
uint64_t last_modified_time;
uint64_t available_capacity_bytes;
uint64_t total_capacity_bytes;
std::optional< peer_id_t > current_leader{std::nullopt};
uint8_t meta[meta_length]{};
uint64_t sealed_lsn{INT64_MAX}; // appended after all v2 fields for forward compat

auto operator<=>(ShardInfo const& rhs) const { return id <=> rhs.id; }
auto operator==(ShardInfo const& rhs) const { return id == rhs.id; }
Expand All @@ -57,7 +58,8 @@ class ShardManager : public Manager< ShardError > {

virtual AsyncResult< ShardInfo > get_shard(shard_id_t id, trace_id_t tid = 0) const = 0;
virtual AsyncResult< InfoList > list_shards(pg_id_t id, trace_id_t tid = 0) const = 0;
virtual AsyncResult< ShardInfo > create_shard(pg_id_t pg_owner, uint64_t size_bytes, std::string meta, trace_id_t tid = 0) = 0;
virtual AsyncResult< ShardInfo > create_shard(pg_id_t pg_owner, uint64_t size_bytes, std::string meta,
trace_id_t tid = 0) = 0;
virtual AsyncResult< ShardInfo > seal_shard(shard_id_t id, trace_id_t tid = 0) = 0;
};

Expand Down
53 changes: 40 additions & 13 deletions src/lib/homestore_backend/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ settings_gen_cpp(
hs_backend_config.fbs
resync_pg_data.fbs
resync_shard_data.fbs
legacy_resync_shard_data.fbs
resync_blob_data.fbs
)

Expand All @@ -55,28 +56,28 @@ add_subdirectory(tests)
add_executable(homestore_test_pg)
target_sources(homestore_test_pg PRIVATE $<TARGET_OBJECTS:homestore_tests_pg>)
target_link_libraries(homestore_test_pg PUBLIC homeobject_homestore ${COMMON_TEST_DEPS})
add_test(NAME HomestoreTestPg COMMAND homestore_test_pg -csv error --executor immediate --config_path ./
add_test(NAME HomestoreTestPg COMMAND homestore_test_pg --executor immediate --config_path ./
--override_config homestore_config.consensus.snapshot_freq_distance:0
--override_config homestore_config.consensus.max_grpc_message_size:138412032)

add_executable(homestore_test_shard)
target_sources(homestore_test_shard PRIVATE $<TARGET_OBJECTS:homestore_tests_shard>)
target_link_libraries(homestore_test_shard PUBLIC homeobject_homestore ${COMMON_TEST_DEPS})
add_test(NAME HomestoreTestShard COMMAND homestore_test_shard -csv error --executor immediate --config_path ./
add_test(NAME HomestoreTestShard COMMAND homestore_test_shard --executor immediate --config_path ./
--override_config homestore_config.consensus.snapshot_freq_distance:0
--override_config homestore_config.consensus.max_grpc_message_size:138412032)

add_executable(homestore_test_blob)
target_sources(homestore_test_blob PRIVATE $<TARGET_OBJECTS:homestore_tests_blob>)
target_link_libraries(homestore_test_blob PUBLIC homeobject_homestore ${COMMON_TEST_DEPS})
add_test(NAME HomestoreTestBlob COMMAND homestore_test_blob -csv error --executor immediate --config_path ./
add_test(NAME HomestoreTestBlob COMMAND homestore_test_blob --executor immediate --config_path ./
--override_config homestore_config.consensus.snapshot_freq_distance:0
--override_config homestore_config.consensus.max_grpc_message_size:138412032)

add_executable(homestore_test_misc)
target_sources(homestore_test_misc PRIVATE $<TARGET_OBJECTS:homestore_tests_misc>)
target_link_libraries(homestore_test_misc PUBLIC homeobject_homestore ${COMMON_TEST_DEPS})
add_test(NAME HomestoreTestMisc COMMAND homestore_test_misc -csv error --executor immediate --config_path ./
add_test(NAME HomestoreTestMisc COMMAND homestore_test_misc --executor immediate --config_path ./
--override_config homestore_config.consensus.snapshot_freq_distance=0
--override_config homestore_config.consensus.max_grpc_message_size=138412032
--override_config hs_backend_config.enable_gc=false)
Expand All @@ -86,7 +87,7 @@ add_executable(homestore_test_dynamic)
target_sources(homestore_test_dynamic PRIVATE $<TARGET_OBJECTS:homestore_tests_dynamic>)
target_link_libraries(homestore_test_dynamic PUBLIC homeobject_homestore ${COMMON_TEST_DEPS})
add_test(NAME HomestoreTestReplaceMember
COMMAND homestore_test_dynamic -csv error --executor immediate --config_path ./
COMMAND homestore_test_dynamic --executor immediate --config_path ./
--override_config homestore_config.consensus.snapshot_freq_distance=0
--override_config homestore_config.generic.repl_dev_cleanup_interval_sec=5
--override_config homestore_config.consensus.max_grpc_message_size=138412032
Expand All @@ -95,7 +96,7 @@ add_test(NAME HomestoreTestReplaceMember
--gtest_filter=HomeObjectFixture.ReplaceMember)
# To test both baseline & incremental resync functionality, we use 13 to minimize the likelihood of it being a divisor of the total LSN (currently 30)
add_test(NAME HomestoreTestReplaceMemberWithBaselineResync
COMMAND homestore_test_dynamic -csv error --executor immediate --config_path ./
COMMAND homestore_test_dynamic --executor immediate --config_path ./
--override_config homestore_config.consensus.snapshot_freq_distance=13
--override_config homestore_config.consensus.num_reserved_log_items=13
--override_config homestore_config.resource_limits.raft_logstore_reserve_threshold=13
Expand All @@ -106,7 +107,7 @@ add_test(NAME HomestoreTestReplaceMemberWithBaselineResync
--override_config homestore_config.consensus.laggy_threshold=2000
--gtest_filter=HomeObjectFixture.ReplaceMember)
add_test(NAME HomestoreTestBaselineResync
COMMAND homestore_test_dynamic -csv error --executor immediate --config_path ./
COMMAND homestore_test_dynamic --executor immediate --config_path ./
--override_config homestore_config.consensus.snapshot_freq_distance=13
--override_config homestore_config.consensus.num_reserved_log_items=13
--override_config homestore_config.resource_limits.raft_logstore_reserve_threshold=13
Expand All @@ -117,7 +118,7 @@ add_test(NAME HomestoreTestBaselineResync
--override_config homestore_config.consensus.laggy_threshold=2000
--gtest_filter=HomeObjectFixture.BaselineResync*)
add_test(NAME HomestoreResyncTestWithFollowerRestart
COMMAND homestore_test_dynamic -csv error --executor immediate --config_path ./
COMMAND homestore_test_dynamic --executor immediate --config_path ./
--override_config homestore_config.consensus.snapshot_freq_distance=13
--override_config homestore_config.consensus.num_reserved_log_items=13
--override_config homestore_config.resource_limits.raft_logstore_reserve_threshold=13
Expand All @@ -128,7 +129,7 @@ add_test(NAME HomestoreResyncTestWithFollowerRestart
--override_config homestore_config.consensus.laggy_threshold=2000
--gtest_filter=HomeObjectFixture.RestartFollower*)
add_test(NAME HomestoreResyncTestWithLeaderRestart
COMMAND homestore_test_dynamic -csv error --executor immediate --config_path ./
COMMAND homestore_test_dynamic --executor immediate --config_path ./
--override_config homestore_config.consensus.snapshot_freq_distance=13
--override_config homestore_config.consensus.num_reserved_log_items=13
--override_config homestore_config.resource_limits.raft_logstore_reserve_threshold=13
Expand All @@ -139,7 +140,7 @@ add_test(NAME HomestoreResyncTestWithLeaderRestart
--override_config homestore_config.consensus.laggy_threshold=2000
--gtest_filter=HomeObjectFixture.RestartLeader*)
#add_test(NAME HomestoreReplaceMemberRollbackTest
# COMMAND homestore_test_dynamic -csv error --executor immediate --config_path ./
# COMMAND homestore_test_dynamic --executor immediate --config_path ./
# --override_config homestore_config.consensus.snapshot_freq_distance=13
# --override_config homestore_config.consensus.num_reserved_log_items=13
# --override_config homestore_config.resource_limits.raft_logstore_reserve_threshold=13
Expand All @@ -152,7 +153,7 @@ add_test(NAME HomestoreResyncTestWithLeaderRestart

# GC tests
add_test(NAME FetchDataWithOriginatorGC
COMMAND homestore_test_dynamic -csv error --executor immediate --config_path ./
COMMAND homestore_test_dynamic --executor immediate --config_path ./
--override_config hs_backend_config.enable_gc=true
--override_config hs_backend_config.gc_enable_read_verify=true
--override_config hs_backend_config.gc_garbage_rate_threshold=50
Expand All @@ -162,9 +163,35 @@ add_test(NAME FetchDataWithOriginatorGC
add_executable(homestore_test_gc)
target_sources(homestore_test_gc PRIVATE $<TARGET_OBJECTS:homestore_tests_gc>)
target_link_libraries(homestore_test_gc PUBLIC homeobject_homestore ${COMMON_TEST_DEPS})
add_test(NAME HomestoreTestGC COMMAND homestore_test_gc -csv error --executor immediate --config_path ./
add_test(NAME HomestoreTestGC COMMAND homestore_test_gc --executor immediate --config_path ./
--override_config hs_backend_config.enable_gc=true
--override_config hs_backend_config.gc_enable_read_verify=true
--override_config hs_backend_config.gc_garbage_rate_threshold=0
--override_config hs_backend_config.gc_garbage_rate_threshold_low=0
--override_config hs_backend_config.gc_scan_interval_sec=5)
--override_config hs_backend_config.gc_scan_interval_sec=5
--gtest_filter=-HomeObjectFixture.StalePChunkRouteAfterGC:-HomeObjectFixture.StaleBlobRouteAfterSealAndGC)

# Shard-race repro tests: require chunks_per_pg=1 (forces vchunk reuse after seal) and gc_garbage_rate_threshold=0.
# Run together since they share the same config requirements.
add_test(NAME HomestoreTestGC_ShardRaceTests
COMMAND homestore_test_gc --executor immediate --config_path ./
--chunks_per_pg 1
--override_config hs_backend_config.enable_gc=true
--override_config hs_backend_config.gc_garbage_rate_threshold=0
--override_config hs_backend_config.gc_garbage_rate_threshold_low=0
--override_config hs_backend_config.gc_scan_interval_sec=5
--gtest_filter=HomeObjectFixture.StalePChunkRouteAfterGC:HomeObjectFixture.StaleBlobRouteAfterSealAndGC)

# All tests share disk, port, and shared-memory resources, so run them serially to avoid parallel conflicts.
set_tests_properties(
HomestoreTestBlob
HomestoreTestMisc
HomestoreTestReplaceMember
HomestoreTestReplaceMemberWithBaselineResync
HomestoreTestBaselineResync
HomestoreResyncTestWithFollowerRestart
HomestoreResyncTestWithLeaderRestart
FetchDataWithOriginatorGC
HomestoreTestGC
HomestoreTestGC_ShardRaceTests
PROPERTIES RUN_SERIAL TRUE)
Loading
Loading