From 70769b625292bfbe886c7d900a029e13a0316d60 Mon Sep 17 00:00:00 2001 From: Gabe Bornstein Date: Tue, 4 Jun 2024 14:08:12 -0400 Subject: [PATCH 01/12] Add State struct and mutex for state object --- conanfile.py | 2 +- src/llfs/page_cache.cpp | 64 ++++++++++++++++++++++------------------- src/llfs/page_cache.hpp | 39 ++++++++++++++----------- 3 files changed, 58 insertions(+), 47 deletions(-) diff --git a/conanfile.py b/conanfile.py index 0b6c173b..0f6c7ece 100644 --- a/conanfile.py +++ b/conanfile.py @@ -69,7 +69,7 @@ def configure(self): def requirements(self): - self.requires("batteries/0.52.4", **VISIBLE) + self.requires("batteries/0.52.5", **VISIBLE) self.requires("boost/1.83.0", **VISIBLE) self.requires("cli11/2.3.2", **VISIBLE) self.requires("glog/0.6.0", **VISIBLE) diff --git a/src/llfs/page_cache.cpp b/src/llfs/page_cache.cpp index 39c60115..9f4842c8 100644 --- a/src/llfs/page_cache.cpp +++ b/src/llfs/page_cache.cpp @@ -111,9 +111,11 @@ PageCache::PageCache(std::vector&& storage_pool, max_page_device_id = std::max(max_page_device_id, arena.device().get_id()); } - // Populate this->page_devices_. + batt::ScopedWriteLock state(this->state_); + + // Populate state.page_devices. // - this->page_devices_.resize(max_page_device_id + 1); + state->page_devices.resize(max_page_device_id + 1); for (PageArena& arena : storage_pool) { const page_device_id_int device_id = arena.device().get_id(); const auto page_size_log2 = batt::log2_ceil(arena.device().page_size()); @@ -131,35 +133,35 @@ PageCache::PageCache(std::vector&& storage_pool, /*name=*/batt::to_string("size_", u64{1} << page_size_log2)); } - BATT_CHECK_EQ(this->page_devices_[device_id], nullptr) + BATT_CHECK_EQ(state->page_devices[device_id], nullptr) << "Duplicate entries found for the same device id!" << BATT_INSPECT(device_id); - this->page_devices_[device_id] = std::make_unique( // + state->page_devices[device_id] = std::make_unique( // std::move(arena), // batt::make_copy(this->cache_slot_pool_by_page_size_log2_[page_size_log2]) // ); // We will sort these later. // - this->page_devices_by_page_size_.emplace_back(this->page_devices_[device_id].get()); + state->page_devices_by_page_size.emplace_back(state->page_devices[device_id].get()); } - BATT_CHECK_EQ(this->page_devices_by_page_size_.size(), storage_pool.size()); + BATT_CHECK_EQ(state->page_devices_by_page_size.size(), storage_pool.size()); // Sort the storage pool by page size (MUST be first). // - std::sort(this->page_devices_by_page_size_.begin(), this->page_devices_by_page_size_.end(), + std::sort(state->page_devices_by_page_size.begin(), state->page_devices_by_page_size.end(), PageSizeOrder{}); // Index the storage pool into groups of arenas by page size. // for (usize size_log2 = 6; size_log2 < kMaxPageSizeLog2; ++size_log2) { - auto iter_pair = std::equal_range(this->page_devices_by_page_size_.begin(), - this->page_devices_by_page_size_.end(), + auto iter_pair = std::equal_range(state->page_devices_by_page_size.begin(), + state->page_devices_by_page_size.end(), PageSize{u32{1} << size_log2}, PageSizeOrder{}); - this->page_devices_by_page_size_log2_[size_log2] = - as_slice(this->page_devices_by_page_size_.data() + - std::distance(this->page_devices_by_page_size_.begin(), iter_pair.first), + state->page_devices_by_page_size_log2[size_log2] = + as_slice(state->page_devices_by_page_size.data() + + std::distance(state->page_devices_by_page_size.begin(), iter_pair.first), as_range(iter_pair).size()); } @@ -253,7 +255,8 @@ batt::Status PageCache::register_page_reader(const PageLayoutId& layout_id, cons // void PageCache::close() { - for (const std::unique_ptr& entry : this->page_devices_) { + batt::ScopedReadLock state(this->state_); + for (const std::unique_ptr& entry : state->page_devices) { if (entry) { entry->arena.halt(); } @@ -264,7 +267,8 @@ void PageCache::close() // void PageCache::join() { - for (const std::unique_ptr& entry : this->page_devices_) { + batt::ScopedReadLock state(this->state_); + for (const std::unique_ptr& entry : state->page_devices) { if (entry) { entry->arena.join(); } @@ -420,14 +424,15 @@ void PageCache::prefetch_hint(PageId page_id) //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -Slice PageCache::all_devices() const +Slice PageCache::all_devices() { - return as_slice(this->page_devices_by_page_size_); + batt::ScopedReadLock state(this->state_); + return as_slice(state->page_devices_by_page_size); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -Slice PageCache::devices_with_page_size(usize size) const +Slice PageCache::devices_with_page_size(usize size) { const usize size_log2 = batt::log2_ceil(size); BATT_CHECK_EQ(size, usize{1} << size_log2) << "page size must be a power of 2"; @@ -437,31 +442,31 @@ Slice PageCache::devices_with_page_size(usize //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -Slice PageCache::devices_with_page_size_log2( - usize size_log2) const +Slice PageCache::devices_with_page_size_log2(usize size_log2) { BATT_CHECK_LT(size_log2, kMaxPageSizeLog2); - - return this->page_devices_by_page_size_log2_[size_log2]; + batt::ScopedReadLock state(this->state_); + return state->page_devices_by_page_size_log2[size_log2]; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -const PageArena& PageCache::arena_for_page_id(PageId page_id) const +const PageArena& PageCache::arena_for_page_id(PageId page_id) { return this->arena_for_device_id(PageIdFactory::get_device_id(page_id)); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -const PageArena& PageCache::arena_for_device_id(page_device_id_int device_id_val) const +const PageArena& PageCache::arena_for_device_id(page_device_id_int device_id_val) { - BATT_CHECK_LT(device_id_val, this->page_devices_.size()) + batt::ScopedReadLock state(this->state_); + BATT_CHECK_LT(device_id_val, state->page_devices.size()) << "the specified page_id's device is not in the storage pool for this cache"; - BATT_CHECK_NOT_NULLPTR(this->page_devices_[device_id_val]); + BATT_CHECK_NOT_NULLPTR(state->page_devices[device_id_val]); - return this->page_devices_[device_id_val]->arena; + return state->page_devices[device_id_val]->arena; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -535,12 +540,13 @@ void PageCache::purge(PageId page_id, u64 callers, u64 job_id) // PageCache::PageDeviceEntry* PageCache::get_device_for_page(PageId page_id) { + batt::ScopedReadLock state(this->state_); const page_device_id_int device_id = PageIdFactory::get_device_id(page_id); - if (BATT_HINT_FALSE(device_id >= this->page_devices_.size())) { + if (BATT_HINT_FALSE(device_id >= state->page_devices.size())) { return nullptr; } - return this->page_devices_[device_id].get(); + return state->page_devices[device_id].get(); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -698,7 +704,7 @@ void PageCache::track_new_page_event(const NewPageTracker& tracker) //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -BoxedSeq PageCache::find_new_page_events(PageId page_id) const +BoxedSeq PageCache::find_new_page_events(PageId page_id) { const isize n = this->history_end_.load(); return batt::as_seq(boost::irange(isize{0}, isize(this->history_.size()))) // diff --git a/src/llfs/page_cache.hpp b/src/llfs/page_cache.hpp index da1f12ec..c76f746a 100644 --- a/src/llfs/page_cache.hpp +++ b/src/llfs/page_cache.hpp @@ -42,6 +42,7 @@ #include #include #include +#include #include #include @@ -189,15 +190,15 @@ class PageCache : public PageLoader Status detach(const boost::uuids::uuid& user_id, slot_offset_type slot_offset); - Slice devices_with_page_size_log2(usize size_log2) const; + Slice devices_with_page_size_log2(usize size_log2); - Slice devices_with_page_size(usize size) const; + Slice devices_with_page_size(usize size); - Slice all_devices() const; + Slice all_devices(); - const PageArena& arena_for_page_id(PageId id_val) const; + const PageArena& arena_for_page_id(PageId id_val); - const PageArena& arena_for_device_id(page_device_id_int device_id_val) const; + const PageArena& arena_for_device_id(page_device_id_int device_id_val); //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // PageLoader interface @@ -236,7 +237,7 @@ class PageCache : public PageLoader bool page_might_contain_key(PageId id, const KeyView& key) const; - BoxedSeq find_new_page_events(PageId page_id) const; + BoxedSeq find_new_page_events(PageId page_id); void track_new_page_event(const NewPageTracker& tracker); @@ -321,19 +322,23 @@ class PageCache : public PageLoader // PageCacheMetrics metrics_; - // The arenas backing up this cache, indexed by device id int. - // - std::vector> page_devices_; + struct State { + // The arenas backing up this cache, indexed by device id int. + // + std::vector> page_devices; - // The contents of `storage_pool_`, sorted by non-decreasing page size. - // - std::vector page_devices_by_page_size_; + // The contents of `storage_pool_`, sorted by non-decreasing page size. + // + std::vector page_devices_by_page_size; - // Slices of `this->storage_pool_` that group arenas by page size (log2). For example, - // `this->arenas_by_size_log2_[12]` is the slice of `this->storage_pool_` comprised of - // PageArenas whose page size is 4096. - // - std::array, kMaxPageSizeLog2> page_devices_by_page_size_log2_; + // Slices of `this->storage_pool_` that group arenas by page size (log2). For example, + // `this->arenas_by_size_log2_[12]` is the slice of `this->storage_pool_` comprised of + // PageArenas whose page size is 4096. + // + std::array, kMaxPageSizeLog2> page_devices_by_page_size_log2; + }; + + batt::ReadWriteMutex state_; // A pool of cache slots for each page size. // From 62481a485737a3cba3b677ba6ca72236d608e05b Mon Sep 17 00:00:00 2001 From: Gabe Bornstein Date: Wed, 5 Jun 2024 13:19:45 -0400 Subject: [PATCH 02/12] Wrap PageCache PageDeviceEntries in a State struct and protect them with a Read/Write Mutex --- src/llfs/page_cache.cpp | 33 ++++++++++++++++++------------- src/llfs/page_cache.hpp | 13 ++++++------ src/llfs/storage_context.test.cpp | 23 +++++++++------------ src/llfs/volume.cpp | 4 ++-- src/llfs/volume.test.cpp | 14 ++++++------- 5 files changed, 44 insertions(+), 43 deletions(-) diff --git a/src/llfs/page_cache.cpp b/src/llfs/page_cache.cpp index 9f4842c8..7a013bf7 100644 --- a/src/llfs/page_cache.cpp +++ b/src/llfs/page_cache.cpp @@ -22,7 +22,7 @@ namespace llfs { //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -usize get_page_size(const PageCache::PageDeviceEntry* entry) +usize get_page_size(std::shared_ptr entry) { return entry ? get_page_size(entry->arena) : 0; } @@ -136,14 +136,14 @@ PageCache::PageCache(std::vector&& storage_pool, BATT_CHECK_EQ(state->page_devices[device_id], nullptr) << "Duplicate entries found for the same device id!" << BATT_INSPECT(device_id); - state->page_devices[device_id] = std::make_unique( // + state->page_devices[device_id] = std::make_shared( // std::move(arena), // batt::make_copy(this->cache_slot_pool_by_page_size_log2_[page_size_log2]) // ); // We will sort these later. // - state->page_devices_by_page_size.emplace_back(state->page_devices[device_id].get()); + state->page_devices_by_page_size.emplace_back(state->page_devices[device_id]); } BATT_CHECK_EQ(state->page_devices_by_page_size.size(), storage_pool.size()); @@ -256,7 +256,7 @@ batt::Status PageCache::register_page_reader(const PageLayoutId& layout_id, cons void PageCache::close() { batt::ScopedReadLock state(this->state_); - for (const std::unique_ptr& entry : state->page_devices) { + for (const std::shared_ptr& entry : state->page_devices) { if (entry) { entry->arena.halt(); } @@ -268,7 +268,7 @@ void PageCache::close() void PageCache::join() { batt::ScopedReadLock state(this->state_); - for (const std::unique_ptr& entry : state->page_devices) { + for (const std::shared_ptr& entry : state->page_devices) { if (entry) { entry->arena.join(); } @@ -306,14 +306,15 @@ StatusOr> PageCache::allocate_page_of_size_log2( LatencyTimer alloc_timer{this->metrics_.allocate_page_alloc_latency}; - Slice device_entries = this->devices_with_page_size_log2(size_log2); + Slice> device_entries = + this->devices_with_page_size_log2(size_log2); // TODO [tastolfi 2021-09-08] If the caller wants to wait, which device should we wait on? First // available? Random? Round-Robin? // for (auto wait_arg : {batt::WaitForResource::kFalse, batt::WaitForResource::kTrue}) { - for (PageDeviceEntry* device_entry : device_entries) { - PageArena& arena = device_entry->arena; + for (std::shared_ptr device_entry : device_entries) { + const PageArena& arena = device_entry->arena; StatusOr page_id = arena.allocator().allocate_page(wait_arg, cancel_token); if (!page_id.ok()) { if (page_id.status() == batt::StatusCode::kResourceExhausted) { @@ -388,7 +389,7 @@ Status PageCache::attach(const boost::uuids::uuid& user_id, slot_offset_type slo } }); - for (PageDeviceEntry* entry : this->all_devices()) { + for (std::shared_ptr entry : this->all_devices()) { BATT_CHECK_NOT_NULLPTR(entry); auto arena_status = entry->arena.allocator().attach_user(user_id, slot_offset); BATT_REQUIRE_OK(arena_status); @@ -404,7 +405,7 @@ Status PageCache::attach(const boost::uuids::uuid& user_id, slot_offset_type slo // Status PageCache::detach(const boost::uuids::uuid& user_id, slot_offset_type slot_offset) { - for (PageDeviceEntry* entry : this->all_devices()) { + for (std::shared_ptr entry : this->all_devices()) { BATT_CHECK_NOT_NULLPTR(entry); auto arena_status = entry->arena.allocator().detach_user(user_id, slot_offset); BATT_REQUIRE_OK(arena_status); @@ -424,15 +425,18 @@ void PageCache::prefetch_hint(PageId page_id) //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -Slice PageCache::all_devices() +std::vector> PageCache::all_devices() { batt::ScopedReadLock state(this->state_); - return as_slice(state->page_devices_by_page_size); + std::vector> devices = + state->page_devices_by_page_size; + return devices; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -Slice PageCache::devices_with_page_size(usize size) +Slice> PageCache::devices_with_page_size( + usize size) { const usize size_log2 = batt::log2_ceil(size); BATT_CHECK_EQ(size, usize{1} << size_log2) << "page size must be a power of 2"; @@ -442,7 +446,8 @@ Slice PageCache::devices_with_page_size(usize //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -Slice PageCache::devices_with_page_size_log2(usize size_log2) +Slice> PageCache::devices_with_page_size_log2( + usize size_log2) { BATT_CHECK_LT(size_log2, kMaxPageSizeLog2); batt::ScopedReadLock state(this->state_); diff --git a/src/llfs/page_cache.hpp b/src/llfs/page_cache.hpp index c76f746a..358fd031 100644 --- a/src/llfs/page_cache.hpp +++ b/src/llfs/page_cache.hpp @@ -190,11 +190,11 @@ class PageCache : public PageLoader Status detach(const boost::uuids::uuid& user_id, slot_offset_type slot_offset); - Slice devices_with_page_size_log2(usize size_log2); + Slice> devices_with_page_size_log2(usize size_log2); - Slice devices_with_page_size(usize size); + Slice> devices_with_page_size(usize size); - Slice all_devices(); + std::vector> all_devices(); const PageArena& arena_for_page_id(PageId id_val); @@ -325,17 +325,18 @@ class PageCache : public PageLoader struct State { // The arenas backing up this cache, indexed by device id int. // - std::vector> page_devices; + std::vector> page_devices; // The contents of `storage_pool_`, sorted by non-decreasing page size. // - std::vector page_devices_by_page_size; + std::vector> page_devices_by_page_size; // Slices of `this->storage_pool_` that group arenas by page size (log2). For example, // `this->arenas_by_size_log2_[12]` is the slice of `this->storage_pool_` comprised of // PageArenas whose page size is 4096. // - std::array, kMaxPageSizeLog2> page_devices_by_page_size_log2; + std::array>, kMaxPageSizeLog2> + page_devices_by_page_size_log2; }; batt::ReadWriteMutex state_; diff --git a/src/llfs/storage_context.test.cpp b/src/llfs/storage_context.test.cpp index f3836afd..bd81af70 100644 --- a/src/llfs/storage_context.test.cpp +++ b/src/llfs/storage_context.test.cpp @@ -40,7 +40,6 @@ TEST(StorageContextTest, GetPageCache) llfs::ScopedIoRing::make_new(llfs::MaxQueueDepth{1024}, llfs::ThreadPoolSize{1}); ASSERT_TRUE(io.ok()) << BATT_INSPECT(io.status()); - // Create a StorageContext. // batt::SharedPtr storage_context = batt::make_shared( @@ -84,7 +83,6 @@ TEST(StorageContextTest, GetPageCache) }, }, }); - BATT_REQUIRE_OK(config_4kb); llfs::StatusOr> config_2mb = @@ -120,22 +118,19 @@ TEST(StorageContextTest, GetPageCache) }); BATT_REQUIRE_OK(config_2mb); - return llfs::OkStatus(); }); ASSERT_TRUE(file_create_status.ok()) << BATT_INSPECT(file_create_status); - llfs::StatusOr> cache = storage_context->get_page_cache(); - ASSERT_TRUE(cache.ok()) << BATT_INSPECT(cache.status()); - ASSERT_NE(*cache, nullptr); - - llfs::Slice devices_4kb = - (*cache)->devices_with_page_size(4 * kKiB); - EXPECT_EQ(devices_4kb.size(), 1u); - - llfs::Slice devices_2mb = - (*cache)->devices_with_page_size(2 * kMiB); - EXPECT_EQ(devices_2mb.size(), 1u); + llfs::StatusOr> cache = storage_context->get_page_cache(); + ASSERT_TRUE(cache.ok()) << BATT_INSPECT(cache.status()); + ASSERT_NE(*cache, nullptr); + llfs::Slice> devices_4kb = + (*cache)->devices_with_page_size(4 * kKiB); + EXPECT_EQ(devices_4kb.size(), 1u); + llfs::Slice> devices_2mb = + (*cache)->devices_with_page_size(2 * kMiB); + EXPECT_EQ(devices_2mb.size(), 1u); } } // namespace diff --git a/src/llfs/volume.cpp b/src/llfs/volume.cpp index 2419c4c7..adb9843c 100644 --- a/src/llfs/volume.cpp +++ b/src/llfs/volume.cpp @@ -161,7 +161,7 @@ u64 Volume::calculate_grant_size(const AppendableJob& appendable) const metadata.ids->recycler_uuid, metadata.ids->trimmer_uuid, }) { - for (PageCache::PageDeviceEntry* entry : cache->all_devices()) { + for (std::shared_ptr entry : cache->all_devices()) { BATT_CHECK_NOT_NULLPTR(entry); const PageArena& arena = entry->arena; Optional attachment = @@ -273,7 +273,7 @@ u64 Volume::calculate_grant_size(const AppendableJob& appendable) const metadata.ids->recycler_uuid, metadata.ids->trimmer_uuid, }) { - for (PageCache::PageDeviceEntry* entry : cache->all_devices()) { + for (std::shared_ptr entry : cache->all_devices()) { BATT_CHECK_NOT_NULLPTR(entry); BATT_REQUIRE_OK(entry->arena.allocator().notify_user_recovered(uuid)); } diff --git a/src/llfs/volume.test.cpp b/src/llfs/volume.test.cpp index 628190ae..47df2da4 100644 --- a/src/llfs/volume.test.cpp +++ b/src/llfs/volume.test.cpp @@ -1599,7 +1599,7 @@ TEST_F(VolumeSimTest, ConcurrentAppendJobs) LLFS_VLOG(1) << "checking ref counts..."; - for (llfs::PageCache::PageDeviceEntry* entry : + for (std::shared_ptr entry : sim.cache()->devices_with_page_size(1 * kKiB)) { BATT_CHECK_NOT_NULLPTR(entry); for (llfs::PageId page_id : page_ids) { @@ -1888,7 +1888,7 @@ void VolumeSimTest::verify_post_recovery_expectations(RecoverySimState& state, if (state.recovered_second_page) { EXPECT_FALSE(state.second_job_will_not_commit); - for (llfs::PageCache::PageDeviceEntry* entry : + for (std::shared_ptr entry : sim.cache()->devices_with_page_size(1 * kKiB)) { BATT_CHECK_NOT_NULLPTR(entry); EXPECT_EQ(entry->arena.allocator().free_pool_size(), this->pages_per_device - 1); @@ -1896,7 +1896,7 @@ void VolumeSimTest::verify_post_recovery_expectations(RecoverySimState& state, ASSERT_TRUE(sim.has_data_for_page_id(state.first_page_id).ok()); EXPECT_TRUE(*sim.has_data_for_page_id(state.first_page_id)); } - for (llfs::PageCache::PageDeviceEntry* entry : + for (std::shared_ptr entry : sim.cache()->devices_with_page_size(2 * kKiB)) { BATT_CHECK_NOT_NULLPTR(entry); EXPECT_EQ(entry->arena.allocator().free_pool_size(), this->pages_per_device - 1); @@ -1904,7 +1904,7 @@ void VolumeSimTest::verify_post_recovery_expectations(RecoverySimState& state, ASSERT_TRUE(sim.has_data_for_page_id(state.second_root_page_id).ok()); EXPECT_TRUE(*sim.has_data_for_page_id(state.second_root_page_id)); } - for (llfs::PageCache::PageDeviceEntry* entry : + for (std::shared_ptr entry : sim.cache()->devices_with_page_size(4 * kKiB)) { BATT_CHECK_NOT_NULLPTR(entry); EXPECT_EQ(entry->arena.allocator().free_pool_size(), this->pages_per_device - 1); @@ -1913,7 +1913,7 @@ void VolumeSimTest::verify_post_recovery_expectations(RecoverySimState& state, EXPECT_TRUE(*sim.has_data_for_page_id(state.third_page_id)); } } else { - for (llfs::PageCache::PageDeviceEntry* entry : + for (std::shared_ptr entry : sim.cache()->devices_with_page_size(1 * kKiB)) { BATT_CHECK_NOT_NULLPTR(entry); EXPECT_EQ(entry->arena.allocator().free_pool_size(), this->pages_per_device - 1); @@ -1921,7 +1921,7 @@ void VolumeSimTest::verify_post_recovery_expectations(RecoverySimState& state, ASSERT_TRUE(sim.has_data_for_page_id(state.first_page_id).ok()); EXPECT_TRUE(*sim.has_data_for_page_id(state.first_page_id)); } - for (llfs::PageCache::PageDeviceEntry* entry : + for (std::shared_ptr entry : sim.cache()->devices_with_page_size(2 * kKiB)) { BATT_CHECK_NOT_NULLPTR(entry); EXPECT_EQ(entry->arena.allocator().free_pool_size(), this->pages_per_device); @@ -1933,7 +1933,7 @@ void VolumeSimTest::verify_post_recovery_expectations(RecoverySimState& state, } } } - for (llfs::PageCache::PageDeviceEntry* entry : + for (std::shared_ptr entry : sim.cache()->devices_with_page_size(4 * kKiB)) { BATT_CHECK_NOT_NULLPTR(entry); EXPECT_EQ(entry->arena.allocator().free_pool_size(), this->pages_per_device); From dffa1ef72e2efa135fc030fb25c84254ab1e13d3 Mon Sep 17 00:00:00 2001 From: Gabe Bornstein Date: Wed, 5 Jun 2024 14:55:23 -0400 Subject: [PATCH 03/12] First pass at exposing PageArenas retrieved by StorageContext to PageCache --- src/llfs/storage_context.cpp | 151 +++++++++++++++++++++++++++++++++++ src/llfs/storage_context.hpp | 13 ++- 2 files changed, 162 insertions(+), 2 deletions(-) diff --git a/src/llfs/storage_context.cpp b/src/llfs/storage_context.cpp index 94bfdfec..b0796dfd 100644 --- a/src/llfs/storage_context.cpp +++ b/src/llfs/storage_context.cpp @@ -94,6 +94,157 @@ Status StorageContext::add_existing_file(const batt::SharedPtr& fil return OkStatus(); } +Status StorageContext::increase_storage_capacity( + const std::filesystem::path& dir_path, u64 increase_capacity, PageSize leaf_size, + PageSizeLog2 leaf_size_log2, PageSize node_size, PageSizeLog2 node_size_log2, + const char* const kPageFileName, unsigned int max_tree_height, unsigned int max_attachments) +{ + // TODO: [Gabe Bornstein 6/3/24] A lot of this code is copy-pasted and could be de-duped. + // + + // Calculate the page counts from the total capacity and TreeOptions. + // + const auto max_in_refs_size_per_leaf = 64 * max_tree_height; + + const auto leaf_page_count = + llfs::PageCount{increase_capacity / (leaf_size + max_in_refs_size_per_leaf)}; + + const auto total_leaf_pages_size = leaf_page_count * leaf_size; + const auto total_node_pages_size = increase_capacity - total_leaf_pages_size; + + const auto node_page_count = llfs::PageCount{total_node_pages_size / node_size}; + + VLOG(1) << BATT_INSPECT(increase_capacity) << BATT_INSPECT(node_page_count) + << BATT_INSPECT(leaf_page_count); + + // Create the page file. + // + Status page_file_status = this->add_new_file( + (dir_path / kPageFileName).string(), + [&](llfs::StorageFileBuilder& builder) -> Status // + { + // Add an arena for node pages. + // + llfs::StatusOr> node_pool_config = + builder.add_object( + llfs::PageArenaConfigOptions{ + .uuid = None, + .page_allocator = + llfs::CreateNewPageAllocator{ + .options = + llfs::PageAllocatorConfigOptions{ + .uuid = llfs::None, + .max_attachments = max_attachments, + .page_count = node_page_count, + .log_device = + llfs::CreateNewLogDeviceWithDefaultSize{ + .uuid = llfs::None, + .pages_per_block_log2 = 1, + }, + .page_size_log2 = node_size_log2, + .page_device = llfs::LinkToNewPageDevice{}, + }, + }, + .page_device = + llfs::CreateNewPageDevice{ + .options = + llfs::PageDeviceConfigOptions{ + .uuid = llfs::None, + .device_id = llfs::None, + .page_count = node_page_count, + .page_size_log2 = node_size_log2, + }, + }, + }); + + BATT_REQUIRE_OK(node_pool_config); + + // Add an arena for leaf pages. + // + llfs::StatusOr> leaf_pool_config = + builder.add_object( + llfs::PageArenaConfigOptions{ + .uuid = None, + .page_allocator = + llfs::CreateNewPageAllocator{ + .options = + llfs::PageAllocatorConfigOptions{ + .uuid = llfs::None, + .max_attachments = max_attachments, + .page_count = leaf_page_count, + .log_device = + llfs::CreateNewLogDeviceWithDefaultSize{ + .uuid = llfs::None, + .pages_per_block_log2 = 1, + }, + .page_size_log2 = leaf_size_log2, + .page_device = llfs::LinkToNewPageDevice{}, + }, + }, + .page_device = + llfs::CreateNewPageDevice{ + .options = + llfs::PageDeviceConfigOptions{ + .uuid = llfs::None, + .device_id = llfs::None, + .page_count = leaf_page_count, + .page_size_log2 = leaf_size_log2, + }, + }, + }); + + BATT_REQUIRE_OK(leaf_pool_config); + + return OkStatus(); + }); + BATT_REQUIRE_OK(page_file_status); + return OkStatus(); +} + +std::vector> StorageContext::retrieve_arenas( + std::vector uuids) +{ + // TODO: [Gabe Bornstein 6/3/24] A lot of this code is copy-pasted and could be de-duped. + // + + // Add Arenas to PageCache. + // + std::vector> arenas; + for (boost::uuids::uuid uuid : uuids) { + batt::SharedPtr p_object_info = this->find_object_by_uuid(uuid); + + if (p_object_info->p_config_slot->tag == PackedConfigSlotBase::Tag::kPageArena) { + const auto& packed_arena_config = + config_slot_cast(p_object_info->p_config_slot.object); + + const std::string base_name = + batt::to_string("PageDevice_", packed_arena_config.page_device_uuid); + + StatusOr arena = this->recover_object( + batt::StaticType{}, uuid, + PageAllocatorRuntimeOptions{ + .scheduler = this->get_scheduler(), + .name = batt::to_string(base_name, "_Allocator"), + }, + [&] { + IoRingLogDriverOptions options; + options.name = batt::to_string(base_name, "_AllocatorLog"); + return options; + }(), + IoRingFileRuntimeOptions{ + .io_ring = this->get_io_ring(), + .use_raw_io = true, + .allow_read = true, + .allow_write = true, + }); + + // BATT_REQUIRE_OK(arena); + arenas.push_back(std::make_shared(std::move(*arena))); + } + } + return arenas; +} + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // void StorageContext::set_page_cache_options(const PageCacheOptions& options) diff --git a/src/llfs/storage_context.hpp b/src/llfs/storage_context.hpp index 0370e738..87506af7 100644 --- a/src/llfs/storage_context.hpp +++ b/src/llfs/storage_context.hpp @@ -102,10 +102,19 @@ class StorageContext : public batt::RefCounted // Status add_existing_file(const batt::SharedPtr& file); + Status increase_storage_capacity(const std::filesystem::path& dir_path, u64 increase_capacity, + PageSize leaf_size, PageSizeLog2 leaf_size_log2, + PageSize node_size, PageSizeLog2 node_size_log2, + const char* const kPageFileName = "pages.llfs", + unsigned int max_tree_height = 10, + unsigned int max_attachments = 32); + + std::vector> retrieve_arenas(std::vector uuids); + // Attempts to recover an object of a given type from this context by uuid. // - // The type of the first argument determines the return type, the tag of the packed config, and - // the type of the extra_options parameter pack. + // The type of the first argument determines the return type, the tag of the packed config, + // and the type of the extra_options parameter pack. // template Date: Thu, 6 Jun 2024 09:28:27 -0400 Subject: [PATCH 04/12] Save uuids of newly added objects in --- src/llfs/storage_context.cpp | 9 ++++++--- src/llfs/storage_context.hpp | 11 +++++------ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/llfs/storage_context.cpp b/src/llfs/storage_context.cpp index b0796dfd..4c812005 100644 --- a/src/llfs/storage_context.cpp +++ b/src/llfs/storage_context.cpp @@ -94,7 +94,7 @@ Status StorageContext::add_existing_file(const batt::SharedPtr& fil return OkStatus(); } -Status StorageContext::increase_storage_capacity( +StatusOr> StorageContext::increase_storage_capacity( const std::filesystem::path& dir_path, u64 increase_capacity, PageSize leaf_size, PageSizeLog2 leaf_size_log2, PageSize node_size, PageSizeLog2 node_size_log2, const char* const kPageFileName, unsigned int max_tree_height, unsigned int max_attachments) @@ -119,6 +119,7 @@ Status StorageContext::increase_storage_capacity( // Create the page file. // + std::vector uuids; Status page_file_status = this->add_new_file( (dir_path / kPageFileName).string(), [&](llfs::StorageFileBuilder& builder) -> Status // @@ -194,11 +195,13 @@ Status StorageContext::increase_storage_capacity( }); BATT_REQUIRE_OK(leaf_pool_config); - + boost::uuids::uuid uuid = (*node_pool_config).get()->uuid; + uuids.push_back(uuid); return OkStatus(); }); + BATT_REQUIRE_OK(page_file_status); - return OkStatus(); + return uuids; } std::vector> StorageContext::retrieve_arenas( diff --git a/src/llfs/storage_context.hpp b/src/llfs/storage_context.hpp index 87506af7..5c213fb2 100644 --- a/src/llfs/storage_context.hpp +++ b/src/llfs/storage_context.hpp @@ -102,12 +102,11 @@ class StorageContext : public batt::RefCounted // Status add_existing_file(const batt::SharedPtr& file); - Status increase_storage_capacity(const std::filesystem::path& dir_path, u64 increase_capacity, - PageSize leaf_size, PageSizeLog2 leaf_size_log2, - PageSize node_size, PageSizeLog2 node_size_log2, - const char* const kPageFileName = "pages.llfs", - unsigned int max_tree_height = 10, - unsigned int max_attachments = 32); + StatusOr> increase_storage_capacity( + const std::filesystem::path& dir_path, u64 increase_capacity, PageSize leaf_size, + PageSizeLog2 leaf_size_log2, PageSize node_size, PageSizeLog2 node_size_log2, + const char* const kPageFileName = "pages.llfs", unsigned int max_tree_height = 10, + unsigned int max_attachments = 32); std::vector> retrieve_arenas(std::vector uuids); From b050dda97abd7bb31ffb72801f860783d9942e52 Mon Sep 17 00:00:00 2001 From: Gabe Bornstein Date: Thu, 6 Jun 2024 15:08:34 -0400 Subject: [PATCH 05/12] Refactor copied code into helper functions --- src/llfs/page_cache.cpp | 131 +++++++++++++++++++---------------- src/llfs/page_cache.hpp | 2 + src/llfs/storage_context.cpp | 119 ++++++++++++++----------------- src/llfs/storage_context.hpp | 5 +- 4 files changed, 130 insertions(+), 127 deletions(-) diff --git a/src/llfs/page_cache.cpp b/src/llfs/page_cache.cpp index 7a013bf7..006c5384 100644 --- a/src/llfs/page_cache.cpp +++ b/src/llfs/page_cache.cpp @@ -104,66 +104,7 @@ PageCache::PageCache(std::vector&& storage_pool, { this->cache_slot_pool_by_page_size_log2_.fill(nullptr); - // Find the maximum page device id value. - // - page_device_id_int max_page_device_id = 0; - for (const PageArena& arena : storage_pool) { - max_page_device_id = std::max(max_page_device_id, arena.device().get_id()); - } - - batt::ScopedWriteLock state(this->state_); - - // Populate state.page_devices. - // - state->page_devices.resize(max_page_device_id + 1); - for (PageArena& arena : storage_pool) { - const page_device_id_int device_id = arena.device().get_id(); - const auto page_size_log2 = batt::log2_ceil(arena.device().page_size()); - - BATT_CHECK_EQ(PageSize{1} << page_size_log2, arena.device().page_size()) - << "Page sizes must be powers of 2!"; - - BATT_CHECK_LT(page_size_log2, kMaxPageSizeLog2); - - // Create a slot pool for this page size if we haven't already done so. - // - if (!this->cache_slot_pool_by_page_size_log2_[page_size_log2]) { - this->cache_slot_pool_by_page_size_log2_[page_size_log2] = PageCacheSlot::Pool::make_new( - /*n_slots=*/this->options_.max_cached_pages_per_size_log2[page_size_log2], - /*name=*/batt::to_string("size_", u64{1} << page_size_log2)); - } - - BATT_CHECK_EQ(state->page_devices[device_id], nullptr) - << "Duplicate entries found for the same device id!" << BATT_INSPECT(device_id); - - state->page_devices[device_id] = std::make_shared( // - std::move(arena), // - batt::make_copy(this->cache_slot_pool_by_page_size_log2_[page_size_log2]) // - ); - - // We will sort these later. - // - state->page_devices_by_page_size.emplace_back(state->page_devices[device_id]); - } - BATT_CHECK_EQ(state->page_devices_by_page_size.size(), storage_pool.size()); - - // Sort the storage pool by page size (MUST be first). - // - std::sort(state->page_devices_by_page_size.begin(), state->page_devices_by_page_size.end(), - PageSizeOrder{}); - - // Index the storage pool into groups of arenas by page size. - // - for (usize size_log2 = 6; size_log2 < kMaxPageSizeLog2; ++size_log2) { - auto iter_pair = std::equal_range(state->page_devices_by_page_size.begin(), - state->page_devices_by_page_size.end(), - PageSize{u32{1} << size_log2}, PageSizeOrder{}); - - state->page_devices_by_page_size_log2[size_log2] = - as_slice(state->page_devices_by_page_size.data() + - std::distance(state->page_devices_by_page_size.begin(), iter_pair.first), - as_range(iter_pair).size()); - } + BATT_CHECK_OK(this->add_arenas(storage_pool)); // Register metrics. // @@ -541,6 +482,76 @@ void PageCache::purge(PageId page_id, u64 callers, u64 job_id) } } +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +batt::Status PageCache::add_arenas(std::vector& arenas) +{ + batt::ScopedWriteLock state(this->state_); + + // Find the maximum page device id value. + // + // TODO: [Gabe Bornstein 6/6/24] Consider, this only looks for the max_page_device_id in `arenas`. + // Do we need to include pre-existing ids in `page_devices` as well? + // + page_device_id_int max_page_device_id = 0; + for (const PageArena& arena : arenas) { + max_page_device_id = std::max(max_page_device_id, arena.device().get_id()); + } + + // Populate state.page_devices. + // + state->page_devices.resize(max_page_device_id + 1); + for (PageArena& arena : arenas) { + const page_device_id_int device_id = arena.device().get_id(); + const auto page_size_log2 = batt::log2_ceil(arena.device().page_size()); + + BATT_CHECK_EQ(PageSize{1} << page_size_log2, arena.device().page_size()) + << "Page sizes must be powers of 2!"; + + BATT_CHECK_LT(page_size_log2, kMaxPageSizeLog2); + + // Create a slot pool for this page size if we haven't already done so. + // + if (!this->cache_slot_pool_by_page_size_log2_[page_size_log2]) { + this->cache_slot_pool_by_page_size_log2_[page_size_log2] = PageCacheSlot::Pool::make_new( + /*n_slots=*/this->options_.max_cached_pages_per_size_log2[page_size_log2], + /*name=*/batt::to_string("size_", u64{1} << page_size_log2)); + } + + BATT_CHECK_EQ(state->page_devices[device_id], nullptr) + << "Duplicate entries found for the same device id!" << BATT_INSPECT(device_id); + + state->page_devices[device_id] = std::make_shared( // + std::move(arena), // + batt::make_copy(this->cache_slot_pool_by_page_size_log2_[page_size_log2]) // + ); + + // We will sort these later. + // + state->page_devices_by_page_size.emplace_back(state->page_devices[device_id]); + } + // BATT_CHECK_EQ(state->page_devices_by_page_size.size(), storage_pool.size()); + + // Sort the storage pool by page size (MUST be first). + // + std::sort(state->page_devices_by_page_size.begin(), state->page_devices_by_page_size.end(), + PageSizeOrder{}); + + // Index the storage pool into groups of arenas by page size. + // + for (usize size_log2 = 6; size_log2 < kMaxPageSizeLog2; ++size_log2) { + auto iter_pair = std::equal_range(state->page_devices_by_page_size.begin(), + state->page_devices_by_page_size.end(), + PageSize{u32{1} << size_log2}, PageSizeOrder{}); + + state->page_devices_by_page_size_log2[size_log2] = + as_slice(state->page_devices_by_page_size.data() + + std::distance(state->page_devices_by_page_size.begin(), iter_pair.first), + as_range(iter_pair).size()); + } + return batt::OkStatus(); +} + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // PageCache::PageDeviceEntry* PageCache::get_device_for_page(PageId page_id) diff --git a/src/llfs/page_cache.hpp b/src/llfs/page_cache.hpp index 358fd031..c8b1789d 100644 --- a/src/llfs/page_cache.hpp +++ b/src/llfs/page_cache.hpp @@ -312,6 +312,8 @@ class PageCache : public PageLoader const Optional& required_layout, OkIfNotFound ok_if_not_found); + batt::Status add_arenas(std::vector& arenas); + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // The configuration passed in at creation time. diff --git a/src/llfs/storage_context.cpp b/src/llfs/storage_context.cpp index 4c812005..119a13fe 100644 --- a/src/llfs/storage_context.cpp +++ b/src/llfs/storage_context.cpp @@ -204,55 +204,68 @@ StatusOr> StorageContext::increase_storage_capac return uuids; } -std::vector> StorageContext::retrieve_arenas( - std::vector uuids) +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +void StorageContext::set_page_cache_options(const PageCacheOptions& options) +{ + this->page_cache_options_ = options; +} + +Status StorageContext::recover_arena(std::vector& arenas, boost::uuids::uuid uuid, + batt::SharedPtr p_object_info) +{ + // TODO: [Gabe Bornstein 6/6/24] Verify that each arena we attempt to recover has not yet been + // recovered. If we recover the same arena multiple times, it may exist in memory multiple times, + // unaware of the other instance, which can cause issues. + // + + if (p_object_info->p_config_slot->tag == PackedConfigSlotBase::Tag::kPageArena) { + const auto& packed_arena_config = + config_slot_cast(p_object_info->p_config_slot.object); + + const std::string base_name = + batt::to_string("PageDevice_", packed_arena_config.page_device_uuid); + + StatusOr arena = this->recover_object( + batt::StaticType{}, uuid, + PageAllocatorRuntimeOptions{ + .scheduler = this->get_scheduler(), + .name = batt::to_string(base_name, "_Allocator"), + }, + [&] { + IoRingLogDriverOptions options; + options.name = batt::to_string(base_name, "_AllocatorLog"); + return options; + }(), + IoRingFileRuntimeOptions{ + .io_ring = this->get_io_ring(), + .use_raw_io = true, + .allow_read = true, + .allow_write = true, + }); + + BATT_REQUIRE_OK(arena); + arenas.emplace_back(std::move(*arena)); + } + return OkStatus(); +} + +Status StorageContext::recover_arenas(std::vector& uuids, + std::vector& arenas) { // TODO: [Gabe Bornstein 6/3/24] A lot of this code is copy-pasted and could be de-duped. // // Add Arenas to PageCache. // - std::vector> arenas; for (boost::uuids::uuid uuid : uuids) { batt::SharedPtr p_object_info = this->find_object_by_uuid(uuid); - if (p_object_info->p_config_slot->tag == PackedConfigSlotBase::Tag::kPageArena) { - const auto& packed_arena_config = - config_slot_cast(p_object_info->p_config_slot.object); - - const std::string base_name = - batt::to_string("PageDevice_", packed_arena_config.page_device_uuid); - - StatusOr arena = this->recover_object( - batt::StaticType{}, uuid, - PageAllocatorRuntimeOptions{ - .scheduler = this->get_scheduler(), - .name = batt::to_string(base_name, "_Allocator"), - }, - [&] { - IoRingLogDriverOptions options; - options.name = batt::to_string(base_name, "_AllocatorLog"); - return options; - }(), - IoRingFileRuntimeOptions{ - .io_ring = this->get_io_ring(), - .use_raw_io = true, - .allow_read = true, - .allow_write = true, - }); - - // BATT_REQUIRE_OK(arena); - arenas.push_back(std::make_shared(std::move(*arena))); - } + BATT_CHECK_OK(this->recover_arena(arenas, uuid, p_object_info)); } - return arenas; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -void StorageContext::set_page_cache_options(const PageCacheOptions& options) -{ - this->page_cache_options_ = options; + // TODO: [Gabe Bornstein 6/6/24] Consider adding new arenas to the page cache here. + // + return OkStatus(); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -267,33 +280,7 @@ StatusOr> StorageContext::get_page_cache() for (const auto& [uuid, p_object_info] : this->index_) { if (p_object_info->p_config_slot->tag == PackedConfigSlotBase::Tag::kPageArena) { - const auto& packed_arena_config = - config_slot_cast(p_object_info->p_config_slot.object); - - const std::string base_name = - batt::to_string("PageDevice_", packed_arena_config.page_device_uuid); - - StatusOr arena = this->recover_object( - batt::StaticType{}, uuid, - PageAllocatorRuntimeOptions{ - .scheduler = this->scheduler_, - .name = batt::to_string(base_name, "_Allocator"), - }, - [&] { - IoRingLogDriverOptions options; - options.name = batt::to_string(base_name, "_AllocatorLog"); - return options; - }(), - IoRingFileRuntimeOptions{ - .io_ring = *this->io_ring_, - .use_raw_io = true, - .allow_read = true, - .allow_write = true, - }); - - BATT_REQUIRE_OK(arena); - - storage_pool.emplace_back(std::move(*arena)); + BATT_CHECK_OK(this->recover_arena(storage_pool, uuid, p_object_info)); } } diff --git a/src/llfs/storage_context.hpp b/src/llfs/storage_context.hpp index 5c213fb2..55bab4ee 100644 --- a/src/llfs/storage_context.hpp +++ b/src/llfs/storage_context.hpp @@ -108,7 +108,10 @@ class StorageContext : public batt::RefCounted const char* const kPageFileName = "pages.llfs", unsigned int max_tree_height = 10, unsigned int max_attachments = 32); - std::vector> retrieve_arenas(std::vector uuids); + Status recover_arena(std::vector& arenas, boost::uuids::uuid uuid, + batt::SharedPtr p_object_info); + + Status recover_arenas(std::vector& uuids, std::vector& arenas); // Attempts to recover an object of a given type from this context by uuid. // From d07acf9fc7f947487ea186afa3de595b10503e9a Mon Sep 17 00:00:00 2001 From: Gabe Bornstein Date: Fri, 7 Jun 2024 10:24:57 -0400 Subject: [PATCH 06/12] add functionality for StorageContext to append to PageArenas to PageCache --- src/llfs/page_cache.cpp | 2 ++ src/llfs/page_cache.hpp | 4 ++-- src/llfs/storage_context.cpp | 25 +++++++++++++++++-------- src/llfs/storage_context.hpp | 11 ++++++----- 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/src/llfs/page_cache.cpp b/src/llfs/page_cache.cpp index 006c5384..8e8c9830 100644 --- a/src/llfs/page_cache.cpp +++ b/src/llfs/page_cache.cpp @@ -487,6 +487,8 @@ void PageCache::purge(PageId page_id, u64 callers, u64 job_id) batt::Status PageCache::add_arenas(std::vector& arenas) { batt::ScopedWriteLock state(this->state_); + // TODO: [Gabe Bornstein 6/6/24] Read over this function and make sure it won't break anything + // with just appending some values vs. creating page_cache from scratch. // Find the maximum page device id value. // diff --git a/src/llfs/page_cache.hpp b/src/llfs/page_cache.hpp index c8b1789d..4d22ecee 100644 --- a/src/llfs/page_cache.hpp +++ b/src/llfs/page_cache.hpp @@ -230,6 +230,8 @@ class PageCache : public PageLoader */ StatusOr put_view(std::shared_ptr&& view, u64 callers, u64 job_id); + batt::Status add_arenas(std::vector& arenas); + //----- --- -- - - - - /** \brief Removes all cached data for the specified page. */ @@ -312,8 +314,6 @@ class PageCache : public PageLoader const Optional& required_layout, OkIfNotFound ok_if_not_found); - batt::Status add_arenas(std::vector& arenas); - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // The configuration passed in at creation time. diff --git a/src/llfs/storage_context.cpp b/src/llfs/storage_context.cpp index 119a13fe..0989efbf 100644 --- a/src/llfs/storage_context.cpp +++ b/src/llfs/storage_context.cpp @@ -94,12 +94,16 @@ Status StorageContext::add_existing_file(const batt::SharedPtr& fil return OkStatus(); } -StatusOr> StorageContext::increase_storage_capacity( +// TODO: [Gabe Bornstein 6/4/24] Could encapsulate some of these params in a new llfs object +// +Status StorageContext::increase_storage_capacity( const std::filesystem::path& dir_path, u64 increase_capacity, PageSize leaf_size, PageSizeLog2 leaf_size_log2, PageSize node_size, PageSizeLog2 node_size_log2, const char* const kPageFileName, unsigned int max_tree_height, unsigned int max_attachments) { - // TODO: [Gabe Bornstein 6/3/24] A lot of this code is copy-pasted and could be de-duped. + // TODO: [Gabe Bornstein 6/3/24] A lot of this code is copy-pasted and could be de-duped. This + // code could potentially be moved into turtle_db. It basically already exists there in + // DB::create. // // Calculate the page counts from the total capacity and TreeOptions. @@ -201,7 +205,13 @@ StatusOr> StorageContext::increase_storage_capac }); BATT_REQUIRE_OK(page_file_status); - return uuids; + + std::vector arenas; + BATT_CHECK_OK(this->recover_arenas(uuids, arenas)); + + BATT_CHECK_NE(this->page_cache_, nullptr); + BATT_CHECK_OK(this->page_cache_->add_arenas(arenas)); + return OkStatus(); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -211,6 +221,8 @@ void StorageContext::set_page_cache_options(const PageCacheOptions& options) this->page_cache_options_ = options; } +// TODO: [Gabe Bornstein 6/7/24] This could probably be a private function. +// Status StorageContext::recover_arena(std::vector& arenas, boost::uuids::uuid uuid, batt::SharedPtr p_object_info) { @@ -250,12 +262,11 @@ Status StorageContext::recover_arena(std::vector& arenas, boost::uuid return OkStatus(); } +// TODO: [Gabe Bornstein 6/7/24] This could probably be a private function. +// Status StorageContext::recover_arenas(std::vector& uuids, std::vector& arenas) { - // TODO: [Gabe Bornstein 6/3/24] A lot of this code is copy-pasted and could be de-duped. - // - // Add Arenas to PageCache. // for (boost::uuids::uuid uuid : uuids) { @@ -263,8 +274,6 @@ Status StorageContext::recover_arenas(std::vector& uuids, BATT_CHECK_OK(this->recover_arena(arenas, uuid, p_object_info)); } - // TODO: [Gabe Bornstein 6/6/24] Consider adding new arenas to the page cache here. - // return OkStatus(); } diff --git a/src/llfs/storage_context.hpp b/src/llfs/storage_context.hpp index 55bab4ee..00382626 100644 --- a/src/llfs/storage_context.hpp +++ b/src/llfs/storage_context.hpp @@ -102,11 +102,12 @@ class StorageContext : public batt::RefCounted // Status add_existing_file(const batt::SharedPtr& file); - StatusOr> increase_storage_capacity( - const std::filesystem::path& dir_path, u64 increase_capacity, PageSize leaf_size, - PageSizeLog2 leaf_size_log2, PageSize node_size, PageSizeLog2 node_size_log2, - const char* const kPageFileName = "pages.llfs", unsigned int max_tree_height = 10, - unsigned int max_attachments = 32); + Status increase_storage_capacity(const std::filesystem::path& dir_path, u64 increase_capacity, + PageSize leaf_size, PageSizeLog2 leaf_size_log2, + PageSize node_size, PageSizeLog2 node_size_log2, + const char* const kPageFileName = "pages.llfs", + unsigned int max_tree_height = 10, + unsigned int max_attachments = 32); Status recover_arena(std::vector& arenas, boost::uuids::uuid uuid, batt::SharedPtr p_object_info); From 43ccf98de1dac77c883ff32bf91f31ba56152b23 Mon Sep 17 00:00:00 2001 From: Gabe Bornstein Date: Thu, 13 Jun 2024 15:57:29 -0400 Subject: [PATCH 07/12] Finish first attempt of increase_storage_capacity and add PageArenas to PageCache --- src/llfs/ioring_impl.cpp | 4 - src/llfs/ioring_log_device_storage.cpp | 2 +- src/llfs/ioring_log_initializer.ipp | 2 - src/llfs/page_allocator_config.cpp | 3 +- src/llfs/page_arena_config.cpp | 1 - src/llfs/page_cache.cpp | 8 + src/llfs/page_cache.hpp | 2 + src/llfs/page_id_factory.hpp | 1 + src/llfs/raw_block_file_impl.cpp | 1 - src/llfs/storage_context.cpp | 43 ++--- src/llfs/storage_context.hpp | 6 +- src/llfs/storage_context.test.cpp | 239 +++++++++++++++---------- src/llfs/storage_file_builder.cpp | 5 +- src/llfs/storage_file_builder.hpp | 3 +- 14 files changed, 189 insertions(+), 131 deletions(-) diff --git a/src/llfs/ioring_impl.cpp b/src/llfs/ioring_impl.cpp index ebea7243..b70b8f1f 100644 --- a/src/llfs/ioring_impl.cpp +++ b/src/llfs/ioring_impl.cpp @@ -160,14 +160,11 @@ Status IoRingImpl::run() noexcept // grabbing/running a completion handler? // constexpr usize kMaxNoopCount = 1000 * 1000; - auto on_scope_exit = batt::finally([this] { LLFS_DVLOG(1) << "IoRingImpl::run() " << BATT_INSPECT(this->work_count_) << " LEAVING"; }); - CompletionHandler* handler = nullptr; usize noop_count = 0; - while (this->can_run()) { LLFS_DVLOG(1) << "IoRingImpl::run() top of loop;" << BATT_INSPECT(this->work_count_); @@ -244,7 +241,6 @@ Status IoRingImpl::run() noexcept } } } - // If we are returning with a ready-to-run handler, stash it for later. // if (handler) { diff --git a/src/llfs/ioring_log_device_storage.cpp b/src/llfs/ioring_log_device_storage.cpp index ab36b2c7..9fcfa950 100644 --- a/src/llfs/ioring_log_device_storage.cpp +++ b/src/llfs/ioring_log_device_storage.cpp @@ -39,7 +39,6 @@ namespace llfs { , caller_{caller} , thread_{[this] { LLFS_VLOG(1) << "(" << this->caller_ << ") invoking IoRing::run()"; - Status io_status = this->storage_.io_ring_.run(); if (!io_status.ok()) { LLFS_LOG_WARNING() << "(" << this->caller_ << ") IoRing::run() returned: " << io_status; @@ -64,6 +63,7 @@ DefaultIoRingLogDeviceStorage::EventLoopTask::~EventLoopTask() noexcept void DefaultIoRingLogDeviceStorage::EventLoopTask::join() { this->join_called_ = true; + BATT_CHECK_OK(this->done_.await_equal(true)); this->thread_.join(); } diff --git a/src/llfs/ioring_log_initializer.ipp b/src/llfs/ioring_log_initializer.ipp index bbb3ce4d..b9937b1c 100644 --- a/src/llfs/ioring_log_initializer.ipp +++ b/src/llfs/ioring_log_initializer.ipp @@ -46,7 +46,6 @@ inline batt::Status BasicIoRingLogInitializer::run() << BATT_INSPECT(this->n_blocks_to_init_); { batt::MutableBuffer memory{this->subtasks_.data(), this->subtasks_.size() * sizeof(Subtask)}; - LLFS_VLOG(1) << "memory = " << (const void*)memory.data() << ".." << (const void*)(this->subtasks_.data() + this->subtasks_.size()); @@ -73,7 +72,6 @@ inline batt::Status BasicIoRingLogInitializer::run() } Status all_finished = this->finished_count_.await_equal(this->subtasks_.size()); BATT_REQUIRE_OK(all_finished); - for (auto& task : this->subtasks_) { BATT_REQUIRE_OK(task.final_status); } diff --git a/src/llfs/page_allocator_config.cpp b/src/llfs/page_allocator_config.cpp index ee738fa0..7d829e24 100644 --- a/src/llfs/page_allocator_config.cpp +++ b/src/llfs/page_allocator_config.cpp @@ -153,8 +153,7 @@ StatusOr> recover_storage_object( const auto page_ids = PageIdFactory{ PageCount{BATT_CHECKED_CAST(PageCount::value_type, p_allocator_config->page_count.value())}, - p_allocator_config->page_device_id, - }; + p_allocator_config->page_device_id}; return PageAllocator::recover(allocator_options, page_ids, **log_factory); } diff --git a/src/llfs/page_arena_config.cpp b/src/llfs/page_arena_config.cpp index a5165499..d9b65a69 100644 --- a/src/llfs/page_arena_config.cpp +++ b/src/llfs/page_arena_config.cpp @@ -192,7 +192,6 @@ StatusOr recover_storage_object( }; BATT_CHECK_EQ(arena.id(), arena.allocator().get_device_id()); - return arena; } diff --git a/src/llfs/page_cache.cpp b/src/llfs/page_cache.cpp index 8e8c9830..2450ec9f 100644 --- a/src/llfs/page_cache.cpp +++ b/src/llfs/page_cache.cpp @@ -554,6 +554,14 @@ batt::Status PageCache::add_arenas(std::vector& arenas) return batt::OkStatus(); } +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +int PageCache::get_num_page_devices() +{ + batt::ScopedReadLock state(this->state_); + return state->page_devices.size(); +} + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // PageCache::PageDeviceEntry* PageCache::get_device_for_page(PageId page_id) diff --git a/src/llfs/page_cache.hpp b/src/llfs/page_cache.hpp index 4d22ecee..7fe5277d 100644 --- a/src/llfs/page_cache.hpp +++ b/src/llfs/page_cache.hpp @@ -232,6 +232,8 @@ class PageCache : public PageLoader batt::Status add_arenas(std::vector& arenas); + int get_num_page_devices(); + //----- --- -- - - - - /** \brief Removes all cached data for the specified page. */ diff --git a/src/llfs/page_id_factory.hpp b/src/llfs/page_id_factory.hpp index 8aa66c8a..58994f56 100644 --- a/src/llfs/page_id_factory.hpp +++ b/src/llfs/page_id_factory.hpp @@ -52,6 +52,7 @@ constexpr page_id_int kPageDeviceIdMask = ((page_id_int{1} << kPageDeviceIdBits) // |<-- kPageDeviceIdBits -->|<-- log_2(Max-Generation-Count) -->|<- log_2(Physical-Page-Count) ->| // +-------------------------+-----------------------------------+--------------------------------+ // + class PageIdFactory : public boost::equality_comparable { public: diff --git a/src/llfs/raw_block_file_impl.cpp b/src/llfs/raw_block_file_impl.cpp index 028478eb..806ab00b 100644 --- a/src/llfs/raw_block_file_impl.cpp +++ b/src/llfs/raw_block_file_impl.cpp @@ -39,7 +39,6 @@ namespace llfs { } }); BATT_REQUIRE_OK(batt::status_from_retval(fd)); - return std::make_unique(IoRing::File{io_ring, fd}); } diff --git a/src/llfs/storage_context.cpp b/src/llfs/storage_context.cpp index 0989efbf..8691d749 100644 --- a/src/llfs/storage_context.cpp +++ b/src/llfs/storage_context.cpp @@ -42,12 +42,10 @@ Status StorageContext::add_existing_named_file(std::string&& file_name, i64 star { StatusOr fd = open_file_read_write(file_name, OpenForAppend{false}, OpenRawIO{true}); BATT_REQUIRE_OK(fd); - IoRingRawBlockFile file{IoRing::File{*this->io_ring_, *fd}}; StatusOr>> config_blocks = read_storage_file(file, start_offset); BATT_REQUIRE_OK(config_blocks); - return this->add_existing_file( batt::make_shared(std::move(file_name), std::move(*config_blocks))); } @@ -64,15 +62,15 @@ Status StorageContext::add_new_file(const std::string& file_name, /*flags=*/O_RDWR | O_CREAT | O_EXCL | O_DIRECT | O_SYNC, /*mode=*/S_IRUSR | S_IWUSR)); - StorageFileBuilder builder{*file, /*base_offset=*/0}; - + llfs::page_device_id_int initial_device_id = + (this->page_cache_) ? this->page_cache_->get_num_page_devices() : 0; + StorageFileBuilder builder{*file, /*base_offset=*/0, initial_device_id}; Status init_status = initializer(builder); if (!init_status.ok()) { file->close().IgnoreError(); delete_file(file_name).IgnoreError(); return init_status; } - Status flush_status = builder.flush_all(); BATT_REQUIRE_OK(flush_status); } @@ -86,11 +84,9 @@ Status StorageContext::add_existing_file(const batt::SharedPtr& fil file->find_all_objects() // | seq::for_each([&](const FileOffsetPtr& slot) { LLFS_VLOG(1) << "Adding " << *slot << " to storage context"; - this->index_.emplace(slot->uuid, batt::make_shared(batt::make_copy(file), slot)); }); - return OkStatus(); } @@ -105,7 +101,6 @@ Status StorageContext::increase_storage_capacity( // code could potentially be moved into turtle_db. It basically already exists there in // DB::create. // - // Calculate the page counts from the total capacity and TreeOptions. // const auto max_in_refs_size_per_leaf = 64 * max_tree_height; @@ -123,7 +118,6 @@ Status StorageContext::increase_storage_capacity( // Create the page file. // - std::vector uuids; Status page_file_status = this->add_new_file( (dir_path / kPageFileName).string(), [&](llfs::StorageFileBuilder& builder) -> Status // @@ -197,18 +191,13 @@ Status StorageContext::increase_storage_capacity( }, }, }); - BATT_REQUIRE_OK(leaf_pool_config); - boost::uuids::uuid uuid = (*node_pool_config).get()->uuid; - uuids.push_back(uuid); return OkStatus(); }); BATT_REQUIRE_OK(page_file_status); - std::vector arenas; - BATT_CHECK_OK(this->recover_arenas(uuids, arenas)); - + BATT_CHECK_OK(this->recover_arenas(arenas)); BATT_CHECK_NE(this->page_cache_, nullptr); BATT_CHECK_OK(this->page_cache_->add_arenas(arenas)); return OkStatus(); @@ -226,18 +215,19 @@ void StorageContext::set_page_cache_options(const PageCacheOptions& options) Status StorageContext::recover_arena(std::vector& arenas, boost::uuids::uuid uuid, batt::SharedPtr p_object_info) { - // TODO: [Gabe Bornstein 6/6/24] Verify that each arena we attempt to recover has not yet been - // recovered. If we recover the same arena multiple times, it may exist in memory multiple times, - // unaware of the other instance, which can cause issues. - // - if (p_object_info->p_config_slot->tag == PackedConfigSlotBase::Tag::kPageArena) { const auto& packed_arena_config = config_slot_cast(p_object_info->p_config_slot.object); + // If we have already recovered a device with the same uuid before, return immediately. If we + // recover the same device twice, would could have asynchronous write conflicts in the future. + // + if (this->recovered_uuids_.find(uuid) != this->recovered_uuids_.end()) { + return OkStatus(); + } + const std::string base_name = batt::to_string("PageDevice_", packed_arena_config.page_device_uuid); - StatusOr arena = this->recover_object( batt::StaticType{}, uuid, PageAllocatorRuntimeOptions{ @@ -256,6 +246,10 @@ Status StorageContext::recover_arena(std::vector& arenas, boost::uuid .allow_write = true, }); + // TODO: [Gabe Bornstein 6/11/24] Consider, should I insert before calling recover_object? + // + this->recovered_uuids_.insert(uuid); + BATT_REQUIRE_OK(arena); arenas.emplace_back(std::move(*arena)); } @@ -264,14 +258,11 @@ Status StorageContext::recover_arena(std::vector& arenas, boost::uuid // TODO: [Gabe Bornstein 6/7/24] This could probably be a private function. // -Status StorageContext::recover_arenas(std::vector& uuids, - std::vector& arenas) +Status StorageContext::recover_arenas(std::vector& arenas) { // Add Arenas to PageCache. // - for (boost::uuids::uuid uuid : uuids) { - batt::SharedPtr p_object_info = this->find_object_by_uuid(uuid); - + for (const auto& [uuid, p_object_info] : this->index_) { BATT_CHECK_OK(this->recover_arena(arenas, uuid, p_object_info)); } return OkStatus(); diff --git a/src/llfs/storage_context.hpp b/src/llfs/storage_context.hpp index 00382626..214fd243 100644 --- a/src/llfs/storage_context.hpp +++ b/src/llfs/storage_context.hpp @@ -112,7 +112,7 @@ class StorageContext : public batt::RefCounted Status recover_arena(std::vector& arenas, boost::uuids::uuid uuid, batt::SharedPtr p_object_info); - Status recover_arenas(std::vector& uuids, std::vector& arenas); + Status recover_arenas(std::vector& arenas); // Attempts to recover an object of a given type from this context by uuid. // @@ -159,6 +159,10 @@ class StorageContext : public batt::RefCounted boost::hash> index_; + // An index of all storage objects by uuid. + // + std::unordered_set> recovered_uuids_; + // Options that will be used to instantiate `this->page_cache_`. // PageCacheOptions page_cache_options_ = PageCacheOptions::with_default_values(); diff --git a/src/llfs/storage_context.test.cpp b/src/llfs/storage_context.test.cpp index bd81af70..3d6d3db3 100644 --- a/src/llfs/storage_context.test.cpp +++ b/src/llfs/storage_context.test.cpp @@ -29,108 +29,167 @@ namespace { using namespace llfs::constants; using namespace llfs::int_types; -TEST(StorageContextTest, GetPageCache) +class StorageContextTest : public ::testing::Test { - const char* storage_file_name = "/tmp/llfs_StorageContextTest_GetPageCache_storage_file.llfs"; - - llfs::delete_file(storage_file_name).IgnoreError(); - EXPECT_FALSE(std::filesystem::exists(std::filesystem::path{storage_file_name})); + public: + void SetUp() override + { + this->dir_name = "/tmp/"; + const char* storage_file_name = "/tmp/llfs_StorageContextTest_GetPageCache_storage_file.llfs"; + + llfs::delete_file(storage_file_name).IgnoreError(); + EXPECT_FALSE(std::filesystem::exists(std::filesystem::path{storage_file_name})); + + this->io = + llfs::ScopedIoRing::make_new(llfs::MaxQueueDepth{1024}, llfs::ThreadPoolSize{1}); + + ASSERT_TRUE(io.ok()) << BATT_INSPECT(io.status()); + // Create a StorageContext. + // + this->storage_context = batt::make_shared( + batt::Runtime::instance().default_scheduler(), io->get_io_ring()); + + boost::uuids::uuid arena_uuid_4kb = llfs::random_uuid(); + boost::uuids::uuid arena_uuid_2mb = llfs::random_uuid(); + + // Create a storage file with two page arenas, one for small pages (4kb), one for large (2mb). + // + llfs::Status file_create_status = storage_context->add_new_file( + storage_file_name, [&](llfs::StorageFileBuilder& builder) -> llfs::Status { + llfs::StatusOr> config_4kb = + builder.add_object(llfs::PageArenaConfigOptions{ + .uuid = arena_uuid_4kb, + .page_allocator = + llfs::CreateNewPageAllocator{ + .options = + llfs::PageAllocatorConfigOptions{ + .uuid = llfs::None, + .max_attachments = 32, + .page_count = llfs::PageCount{32}, + .log_device = + llfs::CreateNewLogDeviceWithDefaultSize{ + .uuid = llfs::None, + .pages_per_block_log2 = + llfs::IoRingLogConfig::kDefaultPagesPerBlockLog2 + 1, + }, + .page_size_log2 = llfs::PageSizeLog2{12}, + .page_device = llfs::LinkToNewPageDevice{}, + }, + }, + .page_device = + llfs::CreateNewPageDevice{ + .options = + llfs::PageDeviceConfigOptions{ + .uuid = llfs::None, + .device_id = llfs::None, + .page_count = llfs::PageCount{32}, + .page_size_log2 = llfs::PageSizeLog2{12}, + }, + }, + }); + BATT_REQUIRE_OK(config_4kb); + + llfs::StatusOr> config_2mb = + builder.add_object(llfs::PageArenaConfigOptions{ + .uuid = arena_uuid_2mb, + .page_allocator = + llfs::CreateNewPageAllocator{ + .options = + llfs::PageAllocatorConfigOptions{ + .uuid = llfs::None, + .max_attachments = 32, + .page_count = llfs::PageCount{32}, + .log_device = + llfs::CreateNewLogDeviceWithDefaultSize{ + .uuid = llfs::None, + .pages_per_block_log2 = + llfs::IoRingLogConfig::kDefaultPagesPerBlockLog2 + 1, + }, + .page_size_log2 = llfs::PageSizeLog2{21}, + .page_device = llfs::LinkToNewPageDevice{}, + }, + }, + .page_device = + llfs::CreateNewPageDevice{ + .options = + llfs::PageDeviceConfigOptions{ + .uuid = llfs::None, + .device_id = llfs::None, + .page_count = llfs::PageCount{32}, + .page_size_log2 = llfs::PageSizeLog2{21}, + }, + }, + }); + + BATT_REQUIRE_OK(config_2mb); + return llfs::OkStatus(); + }); + ASSERT_TRUE(file_create_status.ok()) << BATT_INSPECT(file_create_status); + + llfs::StatusOr> cache = + this->storage_context->get_page_cache(); + ASSERT_TRUE(cache.ok()) << BATT_INSPECT(cache.status()); + ASSERT_NE(*cache, nullptr); + llfs::Slice> devices_4kb = + (*cache)->devices_with_page_size(4 * kKiB); + EXPECT_EQ(devices_4kb.size(), 1u); + llfs::Slice> devices_2mb = + (*cache)->devices_with_page_size(2 * kMiB); + EXPECT_EQ(devices_2mb.size(), 1u); + } - llfs::StatusOr io = - llfs::ScopedIoRing::make_new(llfs::MaxQueueDepth{1024}, llfs::ThreadPoolSize{1}); + batt::SharedPtr storage_context; - ASSERT_TRUE(io.ok()) << BATT_INSPECT(io.status()); - // Create a StorageContext. - // - batt::SharedPtr storage_context = batt::make_shared( - batt::Runtime::instance().default_scheduler(), io->get_io_ring()); + llfs::StatusOr io; - boost::uuids::uuid arena_uuid_4kb = llfs::random_uuid(); - boost::uuids::uuid arena_uuid_2mb = llfs::random_uuid(); + const char* dir_name; +}; - // Create a storage file with two page arenas, one for small pages (4kb), one for large (2mb). +TEST_F(StorageContextTest, GetPageCache) +{ + // This test just runs SetUp() // - llfs::Status file_create_status = storage_context->add_new_file( - storage_file_name, [&](llfs::StorageFileBuilder& builder) -> llfs::Status { - llfs::StatusOr> config_4kb = - builder.add_object(llfs::PageArenaConfigOptions{ - .uuid = arena_uuid_4kb, - .page_allocator = - llfs::CreateNewPageAllocator{ - .options = - llfs::PageAllocatorConfigOptions{ - .uuid = llfs::None, - .max_attachments = 32, - .page_count = llfs::PageCount{32}, - .log_device = - llfs::CreateNewLogDeviceWithDefaultSize{ - .uuid = llfs::None, - .pages_per_block_log2 = - llfs::IoRingLogConfig::kDefaultPagesPerBlockLog2 + 1, - }, - .page_size_log2 = llfs::PageSizeLog2{12}, - .page_device = llfs::LinkToNewPageDevice{}, - }, - }, - .page_device = - llfs::CreateNewPageDevice{ - .options = - llfs::PageDeviceConfigOptions{ - .uuid = llfs::None, - .device_id = llfs::None, - .page_count = llfs::PageCount{32}, - .page_size_log2 = llfs::PageSizeLog2{12}, - }, - }, - }); - BATT_REQUIRE_OK(config_4kb); - - llfs::StatusOr> config_2mb = - builder.add_object(llfs::PageArenaConfigOptions{ - .uuid = arena_uuid_2mb, - .page_allocator = - llfs::CreateNewPageAllocator{ - .options = - llfs::PageAllocatorConfigOptions{ - .uuid = llfs::None, - .max_attachments = 32, - .page_count = llfs::PageCount{32}, - .log_device = - llfs::CreateNewLogDeviceWithDefaultSize{ - .uuid = llfs::None, - .pages_per_block_log2 = - llfs::IoRingLogConfig::kDefaultPagesPerBlockLog2 + 1, - }, - .page_size_log2 = llfs::PageSizeLog2{21}, - .page_device = llfs::LinkToNewPageDevice{}, - }, - }, - .page_device = - llfs::CreateNewPageDevice{ - .options = - llfs::PageDeviceConfigOptions{ - .uuid = llfs::None, - .device_id = llfs::None, - .page_count = llfs::PageCount{32}, - .page_size_log2 = llfs::PageSizeLog2{21}, - }, - }, - }); - - BATT_REQUIRE_OK(config_2mb); - return llfs::OkStatus(); - }); - ASSERT_TRUE(file_create_status.ok()) << BATT_INSPECT(file_create_status); - - llfs::StatusOr> cache = storage_context->get_page_cache(); +} + +TEST_F(StorageContextTest, IncreasePageCacheStorage) +{ + const char* storage_file_name1 = + "llfs_StorageContextTest_GetPageCache_storage_file2.llfs"; + const char* storage_file_name2 = + "llfs_StorageContextTest_GetPageCache_storage_file3.llfs"; + + u8 node_size_log2 = 12 /*4kb*/; + u8 leaf_size_log2 = 21 /*2mb*/; + + BATT_CHECK_OK(this->storage_context->increase_storage_capacity( + this->dir_name, 4096, llfs::PageSize{batt::checked_cast(u64{1} << leaf_size_log2)}, + llfs::PageSizeLog2{leaf_size_log2}, + llfs::PageSize{batt::checked_cast(u64{1} << node_size_log2)}, + llfs::PageSizeLog2{node_size_log2}, storage_file_name1)); + + llfs::StatusOr> cache = + this->storage_context->get_page_cache(); ASSERT_TRUE(cache.ok()) << BATT_INSPECT(cache.status()); ASSERT_NE(*cache, nullptr); llfs::Slice> devices_4kb = (*cache)->devices_with_page_size(4 * kKiB); - EXPECT_EQ(devices_4kb.size(), 1u); + EXPECT_EQ(devices_4kb.size(), 2u); llfs::Slice> devices_2mb = (*cache)->devices_with_page_size(2 * kMiB); - EXPECT_EQ(devices_2mb.size(), 1u); + EXPECT_EQ(devices_2mb.size(), 2u); + + BATT_CHECK_OK(this->storage_context->increase_storage_capacity( + this->dir_name, 4096, llfs::PageSize{batt::checked_cast(u64{1} << leaf_size_log2)}, + llfs::PageSizeLog2{leaf_size_log2}, + llfs::PageSize{batt::checked_cast(u64{1} << node_size_log2)}, + llfs::PageSizeLog2{node_size_log2}, storage_file_name2)); + + ASSERT_TRUE(cache.ok()) << BATT_INSPECT(cache.status()); + ASSERT_NE(*cache, nullptr); + devices_4kb = (*cache)->devices_with_page_size(4 * kKiB); + EXPECT_EQ(devices_4kb.size(), 3u); + devices_2mb = (*cache)->devices_with_page_size(2 * kMiB); + EXPECT_EQ(devices_2mb.size(), 3u); } } // namespace diff --git a/src/llfs/storage_file_builder.cpp b/src/llfs/storage_file_builder.cpp index 3f4f1bb1..d44985c3 100644 --- a/src/llfs/storage_file_builder.cpp +++ b/src/llfs/storage_file_builder.cpp @@ -26,13 +26,14 @@ namespace llfs { //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -StorageFileBuilder::StorageFileBuilder(RawBlockFile& file, i64 base_offset) noexcept +StorageFileBuilder::StorageFileBuilder(RawBlockFile& file, i64 base_offset, + page_device_id_int next_available_device_id) noexcept : file_{file} , config_blocks_{} , unused_payload_{None} , base_offset_{base_offset} , next_offset_{base_offset} - , next_available_device_id_{0} + , next_available_device_id_{next_available_device_id} , pre_flush_actions_{} { } diff --git a/src/llfs/storage_file_builder.hpp b/src/llfs/storage_file_builder.hpp index 22beca4b..009c25f3 100644 --- a/src/llfs/storage_file_builder.hpp +++ b/src/llfs/storage_file_builder.hpp @@ -31,7 +31,8 @@ class StorageFileBuilder //+++++++++++-+-+--+----- --- -- - - - - - explicit StorageFileBuilder(RawBlockFile& file, i64 base_offset) noexcept; + explicit StorageFileBuilder(RawBlockFile& file, i64 base_offset, + page_device_id_int next_available_device_id = 0) noexcept; // Add a storage object to the file. // From 3ccf263935ae365c61d1b6a09d13d21f0ac29da4 Mon Sep 17 00:00:00 2001 From: Gabe Bornstein Date: Tue, 18 Jun 2024 13:43:17 -0400 Subject: [PATCH 08/12] add_arenas -> add_page_devices & add cache_slot_pool_by_page_size_log2 to thread safe struct --- src/llfs/page_cache.cpp | 15 +++++++++------ src/llfs/page_cache.hpp | 22 ++++++++++++---------- src/llfs/storage_context.cpp | 2 +- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/llfs/page_cache.cpp b/src/llfs/page_cache.cpp index 2450ec9f..7ce04750 100644 --- a/src/llfs/page_cache.cpp +++ b/src/llfs/page_cache.cpp @@ -102,9 +102,12 @@ PageCache::PageCache(std::vector&& storage_pool, , metrics_{} , page_readers_{std::make_shared>()} { - this->cache_slot_pool_by_page_size_log2_.fill(nullptr); + { + batt::ScopedWriteLock state(this->state_); + state->cache_slot_pool_by_page_size_log2.fill(nullptr); + } - BATT_CHECK_OK(this->add_arenas(storage_pool)); + BATT_CHECK_OK(this->add_page_devices(storage_pool)); // Register metrics. // @@ -484,7 +487,7 @@ void PageCache::purge(PageId page_id, u64 callers, u64 job_id) //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -batt::Status PageCache::add_arenas(std::vector& arenas) +batt::Status PageCache::add_page_devices(std::vector& arenas) { batt::ScopedWriteLock state(this->state_); // TODO: [Gabe Bornstein 6/6/24] Read over this function and make sure it won't break anything @@ -514,8 +517,8 @@ batt::Status PageCache::add_arenas(std::vector& arenas) // Create a slot pool for this page size if we haven't already done so. // - if (!this->cache_slot_pool_by_page_size_log2_[page_size_log2]) { - this->cache_slot_pool_by_page_size_log2_[page_size_log2] = PageCacheSlot::Pool::make_new( + if (!state->cache_slot_pool_by_page_size_log2[page_size_log2]) { + state->cache_slot_pool_by_page_size_log2[page_size_log2] = PageCacheSlot::Pool::make_new( /*n_slots=*/this->options_.max_cached_pages_per_size_log2[page_size_log2], /*name=*/batt::to_string("size_", u64{1} << page_size_log2)); } @@ -525,7 +528,7 @@ batt::Status PageCache::add_arenas(std::vector& arenas) state->page_devices[device_id] = std::make_shared( // std::move(arena), // - batt::make_copy(this->cache_slot_pool_by_page_size_log2_[page_size_log2]) // + batt::make_copy(state->cache_slot_pool_by_page_size_log2[page_size_log2]) // ); // We will sort these later. diff --git a/src/llfs/page_cache.hpp b/src/llfs/page_cache.hpp index 7fe5277d..bf41a53b 100644 --- a/src/llfs/page_cache.hpp +++ b/src/llfs/page_cache.hpp @@ -230,7 +230,7 @@ class PageCache : public PageLoader */ StatusOr put_view(std::shared_ptr&& view, u64 callers, u64 job_id); - batt::Status add_arenas(std::vector& arenas); + batt::Status add_page_devices(std::vector& arenas); int get_num_page_devices(); @@ -250,16 +250,18 @@ class PageCache : public PageLoader return this->metrics_; } - const PageCacheSlot::Pool::Metrics& metrics_for_page_size(PageSize page_size) const + const PageCacheSlot::Pool::Metrics& metrics_for_page_size(PageSize page_size) { const i32 page_size_log2 = batt::log2_ceil(page_size); + batt::ScopedReadLock state(this->state_); + BATT_CHECK_LT(static_cast(page_size_log2), - this->cache_slot_pool_by_page_size_log2_.size()); + state->cache_slot_pool_by_page_size_log2.size()); - BATT_CHECK_NOT_NULLPTR(this->cache_slot_pool_by_page_size_log2_[page_size_log2]); + BATT_CHECK_NOT_NULLPTR(state->cache_slot_pool_by_page_size_log2[page_size_log2]); - return this->cache_slot_pool_by_page_size_log2_[page_size_log2]->metrics(); + return state->cache_slot_pool_by_page_size_log2[page_size_log2]->metrics(); } private: @@ -341,15 +343,15 @@ class PageCache : public PageLoader // std::array>, kMaxPageSizeLog2> page_devices_by_page_size_log2; + + // A pool of cache slots for each page size. + // + std::array, kMaxPageSizeLog2> + cache_slot_pool_by_page_size_log2; }; batt::ReadWriteMutex state_; - // A pool of cache slots for each page size. - // - std::array, kMaxPageSizeLog2> - cache_slot_pool_by_page_size_log2_; - // A thread-safe shared map from PageLayoutId to PageReader function; layouts must be registered // with the PageCache so that we trace references during page recycling (aka garbage collection). // diff --git a/src/llfs/storage_context.cpp b/src/llfs/storage_context.cpp index 8691d749..7a7fd031 100644 --- a/src/llfs/storage_context.cpp +++ b/src/llfs/storage_context.cpp @@ -199,7 +199,7 @@ Status StorageContext::increase_storage_capacity( std::vector arenas; BATT_CHECK_OK(this->recover_arenas(arenas)); BATT_CHECK_NE(this->page_cache_, nullptr); - BATT_CHECK_OK(this->page_cache_->add_arenas(arenas)); + BATT_CHECK_OK(this->page_cache_->add_page_devices(arenas)); return OkStatus(); } From 6ef6fed367e3706cf9cf404223fabdcedc5b251c Mon Sep 17 00:00:00 2001 From: Gabe Bornstein Date: Mon, 24 Jun 2024 15:27:51 -0400 Subject: [PATCH 09/12] Initial draft of RunOutOfMemory test. Not completely working. --- src/llfs/storage_context.test.cpp | 139 ++++++++++++++++++++++-------- 1 file changed, 103 insertions(+), 36 deletions(-) diff --git a/src/llfs/storage_context.test.cpp b/src/llfs/storage_context.test.cpp index 3d6d3db3..82576043 100644 --- a/src/llfs/storage_context.test.cpp +++ b/src/llfs/storage_context.test.cpp @@ -15,7 +15,9 @@ #include #include +#include #include +#include #include #include @@ -40,8 +42,7 @@ class StorageContextTest : public ::testing::Test llfs::delete_file(storage_file_name).IgnoreError(); EXPECT_FALSE(std::filesystem::exists(std::filesystem::path{storage_file_name})); - this->io = - llfs::ScopedIoRing::make_new(llfs::MaxQueueDepth{1024}, llfs::ThreadPoolSize{1}); + this->io = llfs::ScopedIoRing::make_new(llfs::MaxQueueDepth{1024}, llfs::ThreadPoolSize{1}); ASSERT_TRUE(io.ok()) << BATT_INSPECT(io.status()); // Create a StorageContext. @@ -98,7 +99,7 @@ class StorageContextTest : public ::testing::Test llfs::PageAllocatorConfigOptions{ .uuid = llfs::None, .max_attachments = 32, - .page_count = llfs::PageCount{32}, + .page_count = llfs::PageCount{1}, .log_device = llfs::CreateNewLogDeviceWithDefaultSize{ .uuid = llfs::None, @@ -115,7 +116,7 @@ class StorageContextTest : public ::testing::Test llfs::PageDeviceConfigOptions{ .uuid = llfs::None, .device_id = llfs::None, - .page_count = llfs::PageCount{32}, + .page_count = llfs::PageCount{1}, .page_size_log2 = llfs::PageSizeLog2{21}, }, }, @@ -153,43 +154,109 @@ TEST_F(StorageContextTest, GetPageCache) TEST_F(StorageContextTest, IncreasePageCacheStorage) { - const char* storage_file_name1 = - "llfs_StorageContextTest_GetPageCache_storage_file2.llfs"; - const char* storage_file_name2 = - "llfs_StorageContextTest_GetPageCache_storage_file3.llfs"; + const char* storage_file_name1 = "llfs_StorageContextTest_GetPageCache_storage_file2.llfs"; + const char* storage_file_name2 = "llfs_StorageContextTest_GetPageCache_storage_file3.llfs"; - u8 node_size_log2 = 12 /*4kb*/; - u8 leaf_size_log2 = 21 /*2mb*/; + u8 node_size_log2 = 12 /*4kb*/; + u8 leaf_size_log2 = 21 /*2mb*/; - BATT_CHECK_OK(this->storage_context->increase_storage_capacity( - this->dir_name, 4096, llfs::PageSize{batt::checked_cast(u64{1} << leaf_size_log2)}, - llfs::PageSizeLog2{leaf_size_log2}, - llfs::PageSize{batt::checked_cast(u64{1} << node_size_log2)}, - llfs::PageSizeLog2{node_size_log2}, storage_file_name1)); + llfs::delete_file(storage_file_name1).IgnoreError(); + EXPECT_FALSE(std::filesystem::exists(std::filesystem::path{storage_file_name1})); + llfs::delete_file(storage_file_name2).IgnoreError(); + EXPECT_FALSE(std::filesystem::exists(std::filesystem::path{storage_file_name2})); - llfs::StatusOr> cache = - this->storage_context->get_page_cache(); - ASSERT_TRUE(cache.ok()) << BATT_INSPECT(cache.status()); - ASSERT_NE(*cache, nullptr); - llfs::Slice> devices_4kb = - (*cache)->devices_with_page_size(4 * kKiB); - EXPECT_EQ(devices_4kb.size(), 2u); - llfs::Slice> devices_2mb = - (*cache)->devices_with_page_size(2 * kMiB); - EXPECT_EQ(devices_2mb.size(), 2u); + BATT_CHECK_OK(this->storage_context->increase_storage_capacity( + this->dir_name, 4096, llfs::PageSize{batt::checked_cast(u64{1} << leaf_size_log2)}, + llfs::PageSizeLog2{leaf_size_log2}, + llfs::PageSize{batt::checked_cast(u64{1} << node_size_log2)}, + llfs::PageSizeLog2{node_size_log2}, storage_file_name1)); - BATT_CHECK_OK(this->storage_context->increase_storage_capacity( - this->dir_name, 4096, llfs::PageSize{batt::checked_cast(u64{1} << leaf_size_log2)}, - llfs::PageSizeLog2{leaf_size_log2}, - llfs::PageSize{batt::checked_cast(u64{1} << node_size_log2)}, - llfs::PageSizeLog2{node_size_log2}, storage_file_name2)); + llfs::StatusOr> cache = this->storage_context->get_page_cache(); + ASSERT_TRUE(cache.ok()) << BATT_INSPECT(cache.status()); + ASSERT_NE(*cache, nullptr); + llfs::Slice> devices_4kb = + (*cache)->devices_with_page_size(4 * kKiB); + EXPECT_EQ(devices_4kb.size(), 2u); + llfs::Slice> devices_2mb = + (*cache)->devices_with_page_size(2 * kMiB); + EXPECT_EQ(devices_2mb.size(), 2u); - ASSERT_TRUE(cache.ok()) << BATT_INSPECT(cache.status()); - ASSERT_NE(*cache, nullptr); - devices_4kb = (*cache)->devices_with_page_size(4 * kKiB); - EXPECT_EQ(devices_4kb.size(), 3u); - devices_2mb = (*cache)->devices_with_page_size(2 * kMiB); - EXPECT_EQ(devices_2mb.size(), 3u); + BATT_CHECK_OK(this->storage_context->increase_storage_capacity( + this->dir_name, 4096, llfs::PageSize{batt::checked_cast(u64{1} << leaf_size_log2)}, + llfs::PageSizeLog2{leaf_size_log2}, + llfs::PageSize{batt::checked_cast(u64{1} << node_size_log2)}, + llfs::PageSizeLog2{node_size_log2}, storage_file_name2)); + + ASSERT_TRUE(cache.ok()) << BATT_INSPECT(cache.status()); + ASSERT_NE(*cache, nullptr); + devices_4kb = (*cache)->devices_with_page_size(4 * kKiB); + EXPECT_EQ(devices_4kb.size(), 3u); + devices_2mb = (*cache)->devices_with_page_size(2 * kMiB); + EXPECT_EQ(devices_2mb.size(), 3u); + + llfs::PageCacheMetrics& metrics = (*cache)->metrics(); + LOG(INFO) << "metrics.get_page_view_count: " << metrics.get_page_view_count.load(); + + // TODO: [Gabe Bornstein 6/18/24] Add other invariants to be verified here. + // +} + +TEST_F(StorageContextTest, RunOutOfMemory) +{ + std::string file_name = "llfs_StorageContextTest_RunOutOfMemory_storage_file"; + std::string file_extension = ".llfs"; + // const char* = file_name + ".llfs"; + // const char* storage_file_name2 = "llfs_StorageContextTest_GetPageCache_storage_file5.llfs"; + + u8 node_size_log2 = 12 /*4kb*/; + u8 leaf_size_log2 = 21 /*2mb*/; + u8 num_storage_increases = 100; + u8 num_pages_per_device = 1; + + llfs::StatusOr> cache = this->storage_context->get_page_cache(); + + std::thread new_page_thread([&] { + for (usize i = 0; i < num_pages_per_device * num_storage_increases; ++i) { + LOG(INFO) << "new_page_thread start: " << i; + std::unique_ptr job1 = (*cache)->new_job(); + batt::StatusOr> page1 = + job1->new_page(llfs::PageSize{4096}, batt::WaitForResource::kFalse, + llfs::OpaquePageView::page_layout_id(), llfs::Caller::Unknown, + /*cancel_token=*/llfs::None); + LOG(INFO) << "new_page_thread end: " << i; + } + }); + + std::thread increase_storage_capacity_thread([&] { + for (int i = 0; i < num_storage_increases + 1; ++i) { + LOG(INFO) << "increase_storage_capacity_thread start: " << i; + const char* storage_file_name = (file_name + batt::to_string(i) + file_extension).c_str(); + llfs::delete_file(storage_file_name).IgnoreError(); + EXPECT_FALSE(std::filesystem::exists(std::filesystem::path{storage_file_name})); + + BATT_CHECK_OK(this->storage_context->increase_storage_capacity( + this->dir_name, 4096, llfs::PageSize{batt::checked_cast(u64{1} << leaf_size_log2)}, + llfs::PageSizeLog2{leaf_size_log2}, + llfs::PageSize{batt::checked_cast(u64{1} << node_size_log2)}, + llfs::PageSizeLog2{node_size_log2}, storage_file_name)); + + ASSERT_TRUE(cache.ok()) << BATT_INSPECT(cache.status()); + ASSERT_NE(*cache, nullptr); + llfs::Slice> devices_4kb = + (*cache)->devices_with_page_size(4 * kKiB); + EXPECT_EQ(devices_4kb.size(), i * 1u + 2u); + llfs::Slice> devices_2mb = + (*cache)->devices_with_page_size(2 * kMiB); + EXPECT_EQ(devices_2mb.size(), i * 1u + 2u); + LOG(INFO) << "increase_storage_capacity_thread end: " << i; + } + }); + + llfs::PageCacheMetrics& metrics = (*cache)->metrics(); + LOG(INFO) << "metrics.get_page_view_count: " << metrics.get_page_view_count.load(); + + new_page_thread.join(); + increase_storage_capacity_thread.join(); } } // namespace From 360b01a10fc72bd92a4ae69a7b17198c159fac9e Mon Sep 17 00:00:00 2001 From: Gabe Bornstein Date: Tue, 25 Jun 2024 10:17:22 -0400 Subject: [PATCH 10/12] Finalize tests and changes for back-off-and-retry inside PageCache::allocate_page* --- src/llfs/page_cache.cpp | 238 ++++++++++++++++++------------ src/llfs/page_cache.hpp | 55 ++++--- src/llfs/storage_context.cpp | 2 +- src/llfs/storage_context.test.cpp | 7 +- 4 files changed, 180 insertions(+), 122 deletions(-) diff --git a/src/llfs/page_cache.cpp b/src/llfs/page_cache.cpp index 7ce04750..2e54e31d 100644 --- a/src/llfs/page_cache.cpp +++ b/src/llfs/page_cache.cpp @@ -250,49 +250,82 @@ StatusOr> PageCache::allocate_page_of_size_log2( LatencyTimer alloc_timer{this->metrics_.allocate_page_alloc_latency}; - Slice> device_entries = - this->devices_with_page_size_log2(size_log2); + static const char* kOperationName = "PageCache::allocate_page_of_size_log2"; - // TODO [tastolfi 2021-09-08] If the caller wants to wait, which device should we wait on? First - // available? Random? Round-Robin? + // TODO: [Gabe Bornstein 6/19/24] Tony mentioned there's a different method besides + // `with_retry_policy` that we could use. It could potentially cause less congestion in the + // pipeline of CheckpointGenerator -> Trim -> PageRecycler -> Etc. However, it's more complicated // - for (auto wait_arg : {batt::WaitForResource::kFalse, batt::WaitForResource::kTrue}) { - for (std::shared_ptr device_entry : device_entries) { - const PageArena& arena = device_entry->arena; - StatusOr page_id = arena.allocator().allocate_page(wait_arg, cancel_token); - if (!page_id.ok()) { - if (page_id.status() == batt::StatusCode::kResourceExhausted) { - const u64 page_size = u64{1} << size_log2; - LLFS_LOG_INFO_FIRST_N(1) // - << "Failed to allocate page (pool is empty): " << BATT_INSPECT(page_size); - } - continue; - } - - BATT_CHECK_EQ(PageIdFactory::get_device_id(*page_id), arena.id()); - - LLFS_VLOG(1) << "allocated page " << *page_id; - - this->track_new_page_event(NewPageTracker{ - .ts = 0, - .job_id = job_id, - .page_id = *page_id, - .callers = callers, - .event_id = (int)NewPageTracker::Event::kAllocate, - }); - - // PageDevice::prepare must be thread-safe. - // - return arena.device().prepare(*page_id); - } - - if (wait_for_resource == batt::WaitForResource::kFalse) { - break; - } - } - - LLFS_LOG_WARNING() << "No arena with free space could be found"; - return Status{batt::StatusCode::kUnavailable}; // TODO [tastolfi 2021-10-20] + return // + batt::with_retry_policy( // + // TODO: [Gabe Bornstein 6/20/24] Ensure MAX waittime is < 100 ms + // + batt::ExponentialBackoff{ + .max_attempts = ~u64{0}, + .initial_delay_usec = 500, + .backoff_factor = 3, + .backoff_divisor = 2, + .max_delay_usec = 1000 * 100, + }, + kOperationName, // + [this, &size_log2, &cancel_token, &job_id, &callers, &wait_for_resource] { + this->update_available_pages(); + + batt::StatusOr> page_buffer; + batt::Status s = batt::StatusCode::kResourceExhausted; + page_buffer = s; + + Slice> device_entries = + this->devices_with_page_size_log2(size_log2); + // TODO: [Gabe Bornstein 6/20/24] Currently, we're dropping the state lock in between + // `update_available_pages` and `allocate_page`. Could this cause issues? Could we run + // out of pages? + // + batt::ScopedWriteLock state(this->state_); + // If available_pages does not have not have any pages that match device_entries, + // nothing to do. No available pages. Return and try again. + // + for (std::shared_ptr device_entry : device_entries) { + const PageArena& arena = device_entry->arena; + if (state->arenas_with_available_pages.find(arena.id()) == + state->arenas_with_available_pages.end()) { + continue; + } + StatusOr page_id = + arena.allocator().allocate_page(batt::WaitForResource::kFalse, cancel_token); + if (!page_id.ok()) { + if (page_id.status() == batt::StatusCode::kResourceExhausted) { + const u64 page_size = u64{1} << size_log2; + LLFS_LOG_INFO_FIRST_N(1) // + << "Failed to allocate page (pool is empty): " << BATT_INSPECT(page_size); + } + continue; + } + BATT_CHECK_EQ(PageIdFactory::get_device_id(*page_id), arena.id()); + + LLFS_VLOG(1) << "allocated page " << *page_id; + + this->track_new_page_event(NewPageTracker{ + .ts = 0, + .job_id = job_id, + .page_id = *page_id, + .callers = callers, + .event_id = (int)NewPageTracker::Event::kAllocate, + }); + // PageDevice::prepare must be thread-safe. + // + page_buffer = arena.device().prepare(*page_id); + break; + } + + return page_buffer; + }, // + batt::TaskSleepImpl{}, // + [](const batt::Status& s) -> bool { // + VLOG(2) << "batt::StatusCode::kResourceExhausted == " + << (s == batt::StatusCode::kResourceExhausted); + return batt::status_is_retryable(s) || (s == batt::StatusCode::kResourceExhausted); + }); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -489,71 +522,74 @@ void PageCache::purge(PageId page_id, u64 callers, u64 job_id) // batt::Status PageCache::add_page_devices(std::vector& arenas) { - batt::ScopedWriteLock state(this->state_); - // TODO: [Gabe Bornstein 6/6/24] Read over this function and make sure it won't break anything - // with just appending some values vs. creating page_cache from scratch. + { + batt::ScopedWriteLock state(this->state_); + // TODO: [Gabe Bornstein 6/6/24] Read over this function and make sure it won't break anything + // with just appending some values vs. creating page_cache from scratch. - // Find the maximum page device id value. - // - // TODO: [Gabe Bornstein 6/6/24] Consider, this only looks for the max_page_device_id in `arenas`. - // Do we need to include pre-existing ids in `page_devices` as well? - // - page_device_id_int max_page_device_id = 0; - for (const PageArena& arena : arenas) { - max_page_device_id = std::max(max_page_device_id, arena.device().get_id()); - } + // Find the maximum page device id value. + // + // TODO: [Gabe Bornstein 6/6/24] Consider, this only looks for the max_page_device_id in + // `arenas`. Do we need to include pre-existing ids in `page_devices` as well? + // + page_device_id_int max_page_device_id = 0; + for (const PageArena& arena : arenas) { + max_page_device_id = std::max(max_page_device_id, arena.device().get_id()); + } - // Populate state.page_devices. - // - state->page_devices.resize(max_page_device_id + 1); - for (PageArena& arena : arenas) { - const page_device_id_int device_id = arena.device().get_id(); - const auto page_size_log2 = batt::log2_ceil(arena.device().page_size()); + // Populate state.page_devices. + // + state->page_devices.resize(max_page_device_id + 1); + for (PageArena& arena : arenas) { + const page_device_id_int device_id = arena.device().get_id(); + const auto page_size_log2 = batt::log2_ceil(arena.device().page_size()); - BATT_CHECK_EQ(PageSize{1} << page_size_log2, arena.device().page_size()) - << "Page sizes must be powers of 2!"; + BATT_CHECK_EQ(PageSize{1} << page_size_log2, arena.device().page_size()) + << "Page sizes must be powers of 2!"; - BATT_CHECK_LT(page_size_log2, kMaxPageSizeLog2); + BATT_CHECK_LT(page_size_log2, kMaxPageSizeLog2); - // Create a slot pool for this page size if we haven't already done so. - // - if (!state->cache_slot_pool_by_page_size_log2[page_size_log2]) { - state->cache_slot_pool_by_page_size_log2[page_size_log2] = PageCacheSlot::Pool::make_new( - /*n_slots=*/this->options_.max_cached_pages_per_size_log2[page_size_log2], - /*name=*/batt::to_string("size_", u64{1} << page_size_log2)); - } + // Create a slot pool for this page size if we haven't already done so. + // + if (!state->cache_slot_pool_by_page_size_log2[page_size_log2]) { + state->cache_slot_pool_by_page_size_log2[page_size_log2] = PageCacheSlot::Pool::make_new( + /*n_slots=*/this->options_.max_cached_pages_per_size_log2[page_size_log2], + /*name=*/batt::to_string("size_", u64{1} << page_size_log2)); + } - BATT_CHECK_EQ(state->page_devices[device_id], nullptr) - << "Duplicate entries found for the same device id!" << BATT_INSPECT(device_id); + BATT_CHECK_EQ(state->page_devices[device_id], nullptr) + << "Duplicate entries found for the same device id!" << BATT_INSPECT(device_id); - state->page_devices[device_id] = std::make_shared( // - std::move(arena), // - batt::make_copy(state->cache_slot_pool_by_page_size_log2[page_size_log2]) // - ); + state->page_devices[device_id] = std::make_shared( // + std::move(arena), // + batt::make_copy(state->cache_slot_pool_by_page_size_log2[page_size_log2]) // + ); - // We will sort these later. - // - state->page_devices_by_page_size.emplace_back(state->page_devices[device_id]); - } - // BATT_CHECK_EQ(state->page_devices_by_page_size.size(), storage_pool.size()); + // We will sort these later. + // + state->page_devices_by_page_size.emplace_back(state->page_devices[device_id]); + } + // BATT_CHECK_EQ(state->page_devices_by_page_size.size(), storage_pool.size()); - // Sort the storage pool by page size (MUST be first). - // - std::sort(state->page_devices_by_page_size.begin(), state->page_devices_by_page_size.end(), - PageSizeOrder{}); + // Sort the storage pool by page size (MUST be first). + // + std::sort(state->page_devices_by_page_size.begin(), state->page_devices_by_page_size.end(), + PageSizeOrder{}); - // Index the storage pool into groups of arenas by page size. - // - for (usize size_log2 = 6; size_log2 < kMaxPageSizeLog2; ++size_log2) { - auto iter_pair = std::equal_range(state->page_devices_by_page_size.begin(), - state->page_devices_by_page_size.end(), - PageSize{u32{1} << size_log2}, PageSizeOrder{}); - - state->page_devices_by_page_size_log2[size_log2] = - as_slice(state->page_devices_by_page_size.data() + - std::distance(state->page_devices_by_page_size.begin(), iter_pair.first), - as_range(iter_pair).size()); + // Index the storage pool into groups of arenas by page size. + // + for (usize size_log2 = 6; size_log2 < kMaxPageSizeLog2; ++size_log2) { + auto iter_pair = std::equal_range(state->page_devices_by_page_size.begin(), + state->page_devices_by_page_size.end(), + PageSize{u32{1} << size_log2}, PageSizeOrder{}); + + state->page_devices_by_page_size_log2[size_log2] = + as_slice(state->page_devices_by_page_size.data() + + std::distance(state->page_devices_by_page_size.begin(), iter_pair.first), + as_range(iter_pair).size()); + } } + return batt::OkStatus(); } @@ -623,6 +659,18 @@ auto PageCache::find_page_in_cache(PageId page_id, const Optional& }); } +void PageCache::update_available_pages() +{ + batt::ScopedWriteLock state(this->state_); + for (std::shared_ptr page_device : state->page_devices) { + if (page_device->arena.allocator().free_pool_size() != 0) { + state->arenas_with_available_pages.insert(page_device->arena.id()); + } else { + state->arenas_with_available_pages.erase(page_device->arena.id()); + } + } +} + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // void PageCache::async_load_page_into_slot(const PageCacheSlot::PinnedRef& pinned_slot, diff --git a/src/llfs/page_cache.hpp b/src/llfs/page_cache.hpp index bf41a53b..7afdd77c 100644 --- a/src/llfs/page_cache.hpp +++ b/src/llfs/page_cache.hpp @@ -270,6 +270,35 @@ class PageCache : public PageLoader using PageLayoutReaderMap = std::unordered_map; + struct State { + // The arenas backing up this cache, indexed by device id int. + // + std::vector> page_devices; + + // The contents of `storage_pool_`, sorted by non-decreasing page size. + // + std::vector> page_devices_by_page_size; + + // PageDevices that are not full. If a PageDevice is in this list after `update_available_pages` + // is called, it means that PageDevice has un-used pages. a call to `allocate_page` should + // succeed on that device. + // + std::unordered_set arenas_with_available_pages; + // std::vector> available_pages; + + // Slices of `this->storage_pool_` that group arenas by page size (log2). For example, + // `this->arenas_by_size_log2_[12]` is the slice of `this->storage_pool_` comprised of + // PageArenas whose page size is 4096. + // + std::array>, kMaxPageSizeLog2> + page_devices_by_page_size_log2; + + // A pool of cache slots for each page size. + // + std::array, kMaxPageSizeLog2> + cache_slot_pool_by_page_size_log2; + }; + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - explicit PageCache(std::vector&& storage_pool, @@ -304,6 +333,10 @@ class PageCache : public PageLoader batt::StatusOr find_page_in_cache( PageId page_id, const Optional& required_layout, OkIfNotFound ok_if_not_found); + // Requires that ScopedWriteLock is acquired on state before calling. + // + void update_available_pages(/*batt::ScopedWriteLock& state*/); + //----- --- -- - - - - /** \brief Populates the passed PageCacheSlot asynchronously by attempting to read the page from * storage and setting the Latch value of the slot. @@ -328,28 +361,6 @@ class PageCache : public PageLoader // PageCacheMetrics metrics_; - struct State { - // The arenas backing up this cache, indexed by device id int. - // - std::vector> page_devices; - - // The contents of `storage_pool_`, sorted by non-decreasing page size. - // - std::vector> page_devices_by_page_size; - - // Slices of `this->storage_pool_` that group arenas by page size (log2). For example, - // `this->arenas_by_size_log2_[12]` is the slice of `this->storage_pool_` comprised of - // PageArenas whose page size is 4096. - // - std::array>, kMaxPageSizeLog2> - page_devices_by_page_size_log2; - - // A pool of cache slots for each page size. - // - std::array, kMaxPageSizeLog2> - cache_slot_pool_by_page_size_log2; - }; - batt::ReadWriteMutex state_; // A thread-safe shared map from PageLayoutId to PageReader function; layouts must be registered diff --git a/src/llfs/storage_context.cpp b/src/llfs/storage_context.cpp index 7a7fd031..31e7b3bd 100644 --- a/src/llfs/storage_context.cpp +++ b/src/llfs/storage_context.cpp @@ -61,7 +61,6 @@ Status StorageContext::add_new_file(const std::string& file_name, IoRingRawBlockFile::open(*this->io_ring_, file_name.c_str(), /*flags=*/O_RDWR | O_CREAT | O_EXCL | O_DIRECT | O_SYNC, /*mode=*/S_IRUSR | S_IWUSR)); - llfs::page_device_id_int initial_device_id = (this->page_cache_) ? this->page_cache_->get_num_page_devices() : 0; StorageFileBuilder builder{*file, /*base_offset=*/0, initial_device_id}; @@ -116,6 +115,7 @@ Status StorageContext::increase_storage_capacity( VLOG(1) << BATT_INSPECT(increase_capacity) << BATT_INSPECT(node_page_count) << BATT_INSPECT(leaf_page_count); + LOG(INFO) << "PAGE COUNT add_existing_file: " << node_page_count; // Create the page file. // Status page_file_status = this->add_new_file( diff --git a/src/llfs/storage_context.test.cpp b/src/llfs/storage_context.test.cpp index 82576043..f97364de 100644 --- a/src/llfs/storage_context.test.cpp +++ b/src/llfs/storage_context.test.cpp @@ -205,8 +205,6 @@ TEST_F(StorageContextTest, RunOutOfMemory) { std::string file_name = "llfs_StorageContextTest_RunOutOfMemory_storage_file"; std::string file_extension = ".llfs"; - // const char* = file_name + ".llfs"; - // const char* storage_file_name2 = "llfs_StorageContextTest_GetPageCache_storage_file5.llfs"; u8 node_size_log2 = 12 /*4kb*/; u8 leaf_size_log2 = 21 /*2mb*/; @@ -229,8 +227,9 @@ TEST_F(StorageContextTest, RunOutOfMemory) std::thread increase_storage_capacity_thread([&] { for (int i = 0; i < num_storage_increases + 1; ++i) { - LOG(INFO) << "increase_storage_capacity_thread start: " << i; - const char* storage_file_name = (file_name + batt::to_string(i) + file_extension).c_str(); + std::string file_name_str = file_name + batt::to_string(i) + file_extension; + const char* storage_file_name = file_name_str.c_str(); + LOG(INFO) << "increase_storage_capacity_thread start: " << i << " with file name: " << storage_file_name; llfs::delete_file(storage_file_name).IgnoreError(); EXPECT_FALSE(std::filesystem::exists(std::filesystem::path{storage_file_name})); From a25b4dc5594f8318c9583793964d1ab8c72f4803 Mon Sep 17 00:00:00 2001 From: Gabe Bornstein Date: Mon, 9 Sep 2024 13:19:13 -0400 Subject: [PATCH 11/12] Delete code. We were Adding an Arena to storage context twice. --- src/llfs/page_cache.cpp | 4 ---- src/llfs/storage_context.cpp | 28 ---------------------------- 2 files changed, 32 deletions(-) diff --git a/src/llfs/page_cache.cpp b/src/llfs/page_cache.cpp index 4f007f65..2e54e31d 100644 --- a/src/llfs/page_cache.cpp +++ b/src/llfs/page_cache.cpp @@ -522,7 +522,6 @@ void PageCache::purge(PageId page_id, u64 callers, u64 job_id) // batt::Status PageCache::add_page_devices(std::vector& arenas) { - LOG(INFO) << "Are we calling add_page_devices twice?"; { batt::ScopedWriteLock state(this->state_); // TODO: [Gabe Bornstein 6/6/24] Read over this function and make sure it won't break anything @@ -538,14 +537,11 @@ batt::Status PageCache::add_page_devices(std::vector& arenas) max_page_device_id = std::max(max_page_device_id, arena.device().get_id()); } - LOG(INFO) << "max_page_device_id found: " << max_page_device_id; - // Populate state.page_devices. // state->page_devices.resize(max_page_device_id + 1); for (PageArena& arena : arenas) { const page_device_id_int device_id = arena.device().get_id(); - LOG(INFO) << "Arena has device_id: " << device_id; const auto page_size_log2 = batt::log2_ceil(arena.device().page_size()); BATT_CHECK_EQ(PageSize{1} << page_size_log2, arena.device().page_size()) diff --git a/src/llfs/storage_context.cpp b/src/llfs/storage_context.cpp index c6caecb6..d3d438cd 100644 --- a/src/llfs/storage_context.cpp +++ b/src/llfs/storage_context.cpp @@ -313,34 +313,6 @@ StatusOr> StorageContext::get_page_cache() for (const auto& [uuid, p_object_info] : this->index_) { if (p_object_info->p_config_slot->tag == PackedConfigSlotBase::Tag::kPageArena) { BATT_CHECK_OK(this->recover_arena(storage_pool, uuid, p_object_info)); - - const auto& packed_arena_config = - config_slot_cast(p_object_info->p_config_slot.object); - - const std::string base_name = - batt::to_string("PageDevice_", packed_arena_config.page_device_uuid); - - StatusOr arena = this->recover_object( - batt::StaticType{}, uuid, - PageAllocatorRuntimeOptions{ - .scheduler = this->scheduler_, - .name = batt::to_string(base_name, "_Allocator"), - }, - [&] { - LogDeviceRuntimeOptions options; - options.name = batt::to_string(base_name, "_AllocatorLog"); - return options; - }(), - IoRingFileRuntimeOptions{ - .io_ring = *this->io_ring_, - .use_raw_io = true, - .allow_read = true, - .allow_write = true, - }); - - BATT_REQUIRE_OK(arena); - - storage_pool.emplace_back(std::move(*arena)); } } From ec1013211f085f2ef207e946c7de42d35e4d46b7 Mon Sep 17 00:00:00 2001 From: Gabe Bornstein Date: Mon, 9 Sep 2024 13:21:55 -0400 Subject: [PATCH 12/12] Stop tracking output.txt. I accidentally added it --- output.txt | 758 ----------------------------------------------------- 1 file changed, 758 deletions(-) delete mode 100644 output.txt diff --git a/output.txt b/output.txt deleted file mode 100644 index fd9d1202..00000000 --- a/output.txt +++ /dev/null @@ -1,758 +0,0 @@ -/home/gbornste/Desktop/Projects/MPS/llfs -sending incremental file list -output.txt -deleting .vscode/tasks.json -deleting .vscode/launch.json -deleting .vscode/c_cpp_properties.json -deleting src/CMakeUserPresets.json -.vscode/ -src/ -src/llfs/page_cache.cpp - -sent 32,916 bytes received 597 bytes 67,026.00 bytes/sec -total size is 15,792,275 speedup is 471.23 -gbornste -huponexit on -Docker image: registry.gitlab.com/batteriescpp/batteries:v0.50.7.linux_gcc11_amd64 -CONAN_USER_HOME=/ssh:a8-build-host:/home/gbornste/.a8.conan -XDG_CACHE_HOME=/ssh:a8-build-host:/home/gbornste/.a8.conan/.cache -CONAN_HOME=/ssh:a8-build-host:/home/gbornste/.a8.conan/.conan_2.0.16 -default -batteriesincluded: https://batteriesincluded.cc/artifactory/api/conan/conan-local [Verify SSL: True, Enabled: True] -Setting Conan User... TODO [tastolfi 2023-08-10] fix these commands for Conan 2! -(Setup DONE) -conan-targets.mk: Detected Conan Version==2 -CONAN_CONFIG_FLAGS is --profile "/etc/conan_profile.default" -s build_type=RelWithDebInfo -TCMALLOC_ENV= -rm -rf "/ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/build/RelWithDebInfo" -rm -rf "/ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/test_package/build" -(test -f /setup-conan.sh && /setup-conan.sh || echo "Using ambient Conan config (/setup-conan.sh not found)") -default -batteriesincluded: https://batteriesincluded.cc/artifactory/api/conan/conan-local [Verify SSL: True, Enabled: True] -mkdir -p "/ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/build/RelWithDebInfo" -(cd "/ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/build/RelWithDebInfo" && BATT_BUILD_TESTS=1 conan install --profile "/etc/conan_profile.default" -s build_type=RelWithDebInfo --build=missing "/ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs") - -======== Input profiles ======== -Profile host: -[settings] -arch=x86_64 -build_type=RelWithDebInfo -compiler=gcc -compiler.cppstd=20 -compiler.libcxx=libstdc++11 -compiler.version=11 -os=Linux -[tool_requires] -&: cmake/3.26.4 -[buildenv] -CXXFLAGS=-fno-omit-frame-pointer -CFLAGS=-fno-omit-frame-pointer - -Profile build: -[settings] -arch=x86_64 -build_type=Release -compiler=gcc -compiler.cppstd=gnu17 -compiler.libcxx=libstdc++11 -compiler.version=11 -os=Linux - - -======== Computing dependency graph ======== -Graph root - conanfile.py (llfs/0.1.0-devel): /ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/conanfile.py -Requirements - abseil/20240116.2#54f81a20ccd26a6558e18d57059847e2 - Cache - batteries/0.55.1#ea02793f9c1e7b3bc24b7ff99a9a0a34 - Cache - boost/1.83.0#fd341e07fe362c605d187ed592f2fce8 - Cache - bzip2/1.0.8#d00dac990f08d991998d624be81a9526 - Cache - cli11/2.3.2#1424b9b1d9e3682a7122f415b078b4d7 - Cache - gflags/2.2.2#48d1262ffac8d30c3224befb8275a533 - Cache - glog/0.6.0#f25568477a1100349c4c2959ce946717 - Cache - gtest/1.14.0#25e2a474b4d1aecf5ff4f0555dcdf72c - Cache - libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1 - Cache - libfuse/3.16.2#b426f95ed62675cf8fb54bc407101696 - Cache - libunwind/1.7.2#4f36792edb3ed2c6c6525397f5643c66 - Cache - liburing/2.4#463d05ef1b3465d36eba39a7f2844df3 - Cache - openssl/3.2.0#d52f6cbf28a54701393831929f47dbbd - Cache - pcg-cpp/cci.20220409#b3c4e07c6ef6c2b5ddf2886f8a637bc7 - Cache - protobuf/5.27.0#ccce9aa25886556c6d66c77b2be4d806 - Cache - zlib/1.3#5c0f3a1a222eebb6bff34980bcd3e024 - Cache -Test requirements - gtest/cci.20210126#dafbdf84b58cd687075ace7314651c1a - Cache -Build requirements - b2/5.2.1#91bc73931a0acb655947a81569ed8b80 - Cache - cmake/3.26.4#0b40747e190e755932767f2ec4768ff5 - Cache - cmake/3.30.1#6d832cf2d46f6ec969ca5ed5b41f91eb - Cache - meson/1.2.0#68ba3b73f547af9c46816b2ef842a99d - Cache - ninja/1.12.1#fd583651bf0c6a901943495d49878803 - Cache -Resolved version ranges - b2/[>=5.2 <6]: b2/5.2.1 - cmake/[>=3.16 <4]: cmake/3.30.1 - gtest/[>=1.14.0]: gtest/cci.20210126 - ninja/[>=1.10.2 <2]: ninja/1.12.1 - pcg-cpp/[>=cci.20220409]: pcg-cpp/cci.20220409 - protobuf/[>=3.21.12]: protobuf/5.27.0 -Overrides - zlib/[>=1.2.11 <2]: ['zlib/1.3'] - libunwind/1.8.0: ['libunwind/1.7.2'] - -======== Computing necessary packages ======== -abseil/20240116.2: Checking 7 compatible configurations -abseil/20240116.2: Main binary package '427ba685c637d4b02789cda2bd590c754eb03890' missing. Using compatible package 'e951c2cb74ff982f51710131ee36b3e28f8e4563': compiler.cppstd=17 -gflags/2.2.2: Checking 11 compatible configurations -gflags/2.2.2: Main binary package '44eabc4771aeee384507fc08c863c6435178263c' missing. Using compatible package '3d216c4a794ad4ead4d22073c6f5e44018989263': compiler.cppstd=17 -gtest/1.14.0: Checking 7 compatible configurations -gtest/1.14.0: Main binary package 'dec6797a53c8ee92bc1c6586540cd6f2972efe5b' missing. Using compatible package '8c79091f8481612ee4242f0f3ef3cc13a5a12800': compiler.cppstd=17 -gtest/cci.20210126: Checking 9 compatible configurations -gtest/cci.20210126: Main binary package 'f4512038026b92eb5f53a1892437be6d7a527c40' missing. Using compatible package 'ad418b1d6effc6877f158337ee28e1f5cb0fc13a': compiler.cppstd=17 -protobuf/5.27.0: Checking 7 compatible configurations -protobuf/5.27.0: Main binary package 'fe688174d9d3f9d41ff7419b12114d243c3f8481' missing. Using compatible package '5fe12e7b087c0c4e0ee89ed683371984f642bce4': compiler.cppstd=17 -glog/0.6.0: Checking 11 compatible configurations -glog/0.6.0: Main binary package '00615d8b4cc96735b0723ca020d03b8b83bd7540' missing. Using compatible package 'b87780c0fc63b82ede7a99db267fd11711444420': compiler.cppstd=17 -batteries/0.55.1: Checking 11 compatible configurations -batteries/0.55.1: Main binary package '9d861a1e8ed014810eb7ff93c7bc5679e59011bd' missing. Using compatible package 'd77d782ece1b8314e57db1b08eb99bf84b8ced68': compiler.cppstd=17 -Requirements - abseil/20240116.2#54f81a20ccd26a6558e18d57059847e2:e951c2cb74ff982f51710131ee36b3e28f8e4563#00d86a5fbcd989bb64c057f821f767a8 - Cache - batteries/0.55.1#ea02793f9c1e7b3bc24b7ff99a9a0a34:d77d782ece1b8314e57db1b08eb99bf84b8ced68#e21b5a821423d65308ac97cf15f3501a - Cache - boost/1.83.0#fd341e07fe362c605d187ed592f2fce8:f7b96315a752d5ea8ef3c2eec56915a9bd765c81#9c7df5af22e95b30dc8744ea52d5d940 - Cache - bzip2/1.0.8#d00dac990f08d991998d624be81a9526:d2fd0bf682562652eedb970d66e83aded80b3fb1#6cf5c17b419f33a0a925d1d8e281d3cd - Cache - cli11/2.3.2#1424b9b1d9e3682a7122f415b078b4d7:da39a3ee5e6b4b0d3255bfef95601890afd80709#e04bf357d9bd58bd3f1042ff121bf921 - Cache - gflags/2.2.2#48d1262ffac8d30c3224befb8275a533:3d216c4a794ad4ead4d22073c6f5e44018989263#908a0eb0138f96b6dc4ed65828b96779 - Cache - glog/0.6.0#f25568477a1100349c4c2959ce946717:b87780c0fc63b82ede7a99db267fd11711444420#207682f0e58e96330acab25ad6d79832 - Cache - gtest/1.14.0#25e2a474b4d1aecf5ff4f0555dcdf72c:8c79091f8481612ee4242f0f3ef3cc13a5a12800#c3336292a69ce58cecd28ce76179735c - Cache - libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1:7141b46dc36f162c1999825c09b1ec68c17d6616#6e39386f8f08273ee4c765ac49c8dde2 - Cache - libfuse/3.16.2#b426f95ed62675cf8fb54bc407101696:7141b46dc36f162c1999825c09b1ec68c17d6616#1422880a721df66ec7d9a88e9f180972 - Cache - libunwind/1.7.2#4f36792edb3ed2c6c6525397f5643c66:76ad123c0b4672150480ad00bc8c5ebde5faacd0#bffcb69ae1800b801f12ab5f4201ad27 - Cache - liburing/2.4#463d05ef1b3465d36eba39a7f2844df3:893e43a1ce9cc27ea3549b60ceba7dccc92f04f9#4d36a32e2db28cd63a6f458013bf344a - Cache - openssl/3.2.0#d52f6cbf28a54701393831929f47dbbd:f7124f4882e8a147ba488837fa6a04d4bb37cbf7#f8082f47bb78718fa349b20379316359 - Cache - pcg-cpp/cci.20220409#b3c4e07c6ef6c2b5ddf2886f8a637bc7:da39a3ee5e6b4b0d3255bfef95601890afd80709#4703807cd2cb00aba979bcd262eff030 - Cache - protobuf/5.27.0#ccce9aa25886556c6d66c77b2be4d806:5fe12e7b087c0c4e0ee89ed683371984f642bce4#d806df8d48a82f04534cf590d741597c - Cache - zlib/1.3#5c0f3a1a222eebb6bff34980bcd3e024:7141b46dc36f162c1999825c09b1ec68c17d6616#3e28b9b20446b1f05240c36c569bb31d - Cache -Test requirements -Build requirements - cmake/3.26.4#0b40747e190e755932767f2ec4768ff5:63fead0844576fc02943e16909f08fcdddd6f44b#0e341efd7f3d724d3a793aacff5b86a3 - Cache -Skipped binaries - gtest/cci.20210126, b2/5.2.1, cmake/3.30.1, meson/1.2.0, ninja/1.12.1 - -======== Installing packages ======== -abseil/20240116.2: Already installed! (1 of 17) -bzip2/1.0.8: Already installed! (2 of 17) -cli11/2.3.2: Already installed! (3 of 17) -cmake/3.26.4: Already installed! (4 of 17) -cmake/3.26.4: Appending PATH environment variable: /ssh:a8-build-host:/home/gbornste/.a8.conan/.conan_2.0.16/p/cmake5aa963403baef/p/bin -gflags/2.2.2: Already installed! (5 of 17) -gtest/1.14.0: Already installed! (6 of 17) -libbacktrace/cci.20210118: Already installed! (7 of 17) -liburing/2.4: Already installed! (8 of 17) -pcg-cpp/cci.20220409: Already installed! (9 of 17) -zlib/1.3: Already installed! (10 of 17) -libfuse/3.16.2: Already installed! (11 of 17) -boost/1.83.0: Already installed! (12 of 17) -libunwind/1.7.2: Already installed! (13 of 17) -openssl/3.2.0: Already installed! (14 of 17) -protobuf/5.27.0: Already installed! (15 of 17) -glog/0.6.0: Already installed! (16 of 17) -batteries/0.55.1: Already installed! (17 of 17) -WARN: deprecated: Usage of deprecated Conan 1.X features that will be removed in Conan 2.X: -WARN: deprecated: 'cpp_info.names' used in: abseil/20240116.2, gtest/1.14.0, zlib/1.3, boost/1.83.0, bzip2/1.0.8, protobuf/5.27.0, openssl/3.2.0, batteries/0.55.1, cli11/2.3.2 -WARN: deprecated: 'cpp_info.build_modules' used in: abseil/20240116.2, gflags/2.2.2, protobuf/5.27.0, bzip2/1.0.8, openssl/3.2.0 -WARN: deprecated: 'env_info' used in: cmake/3.26.4, boost/1.83.0, protobuf/5.27.0, bzip2/1.0.8, openssl/3.2.0 -WARN: deprecated: 'cpp_info.filenames' used in: boost/1.83.0, protobuf/5.27.0 -WARN: deprecated: 'user_info' used in: boost/1.83.0 - -======== Finalizing install (deploy, generators) ======== -conanfile.py (llfs/0.1.0-devel): Calling generate() -conanfile.py (llfs/0.1.0-devel): Generators folder: /ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/build/RelWithDebInfo/generators -conanfile.py (llfs/0.1.0-devel): CMakeToolchain generated: conan_toolchain.cmake -conanfile.py (llfs/0.1.0-devel): Preset 'conan-relwithdebinfo' added to CMakePresets.json. Invoke it manually using 'cmake --preset conan-relwithdebinfo' if using CMake>=3.23 -conanfile.py (llfs/0.1.0-devel): If your CMake version is not compatible with CMakePresets (<3.23) call cmake like: 'cmake -G Ninja -DCMAKE_TOOLCHAIN_FILE=/ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/build/RelWithDebInfo/generators/conan_toolchain.cmake -DCMAKE_POLICY_DEFAULT_CMP0091=NEW -DCMAKE_BUILD_TYPE=RelWithDebInfo' -conanfile.py (llfs/0.1.0-devel): CMakeToolchain generated: CMakePresets.json -conanfile.py (llfs/0.1.0-devel): CMakeToolchain generated: ../../../src/CMakeUserPresets.json -conanfile.py (llfs/0.1.0-devel): CMakeDeps necessary find_package() and targets for your CMakeLists.txt - find_package(batteries) - find_package(Boost) - find_package(CLI11) - find_package(glog) - find_package(GTest) - find_package(libbacktrace) - find_package(OpenSSL) - find_package(liburing) - find_package(libfuse) - target_link_libraries(... batteries::batteries boost::boost CLI11::CLI11 glog::glog gtest::gtest libbacktrace::libbacktrace openssl::openssl liburing::liburing libfuse::libfuse) - -======== Generating conan_find_requirements.cmake ======== - -batteries/0.55.1, Traits: build=False, headers=True, libs=True, run=False, visible=True -boost/1.83.0, Traits: build=False, headers=True, libs=True, run=False, visible=True -cli11/2.3.2, Traits: build=False, headers=True, libs=False, run=False, visible=True -glog/0.6.0, Traits: build=False, headers=True, libs=True, run=False, visible=True -gtest/1.14.0, Traits: build=False, headers=True, libs=True, run=False, visible=True -libbacktrace/cci.20210118, Traits: build=False, headers=True, libs=True, run=False, visible=True -openssl/3.2.0, Traits: build=False, headers=True, libs=True, run=False, visible=True -zlib/1.3, Traits: build=False, headers=True, libs=True, run=False, visible=True -... skipping direct=False requirement -liburing/2.4, Traits: build=False, headers=True, libs=True, run=False, visible=True -libfuse/3.16.2, Traits: build=False, headers=True, libs=True, run=False, visible=True -libunwind/1.7.2, Traits: build=False, headers=True, libs=True, run=False, visible=True -... skipping direct=False requirement -cmake/3.26.4, Traits: build=True, headers=False, libs=False, run=True, visible=False -... skipping build=True requirement - -Saved file /ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/build/RelWithDebInfo/conan_find_requirements.cmake - -conanfile.py (llfs/0.1.0-devel): Generating aggregated env files -conanfile.py (llfs/0.1.0-devel): Generated aggregated env files: ['conanbuild.sh', 'conanrun.sh'] -Install finished successfully -(cd "/ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/build/RelWithDebInfo" && BATT_BUILD_TESTS=1 conan build --profile "/etc/conan_profile.default" -s build_type=RelWithDebInfo "/ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs") - -======== Input profiles ======== -Profile host: -[settings] -arch=x86_64 -build_type=RelWithDebInfo -compiler=gcc -compiler.cppstd=20 -compiler.libcxx=libstdc++11 -compiler.version=11 -os=Linux -[tool_requires] -&: cmake/3.26.4 -[buildenv] -CXXFLAGS=-fno-omit-frame-pointer -CFLAGS=-fno-omit-frame-pointer - -Profile build: -[settings] -arch=x86_64 -build_type=Release -compiler=gcc -compiler.cppstd=gnu17 -compiler.libcxx=libstdc++11 -compiler.version=11 -os=Linux - - -======== Computing dependency graph ======== -Graph root - conanfile.py (llfs/0.1.0-devel): /ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/conanfile.py -Requirements - abseil/20240116.2#54f81a20ccd26a6558e18d57059847e2 - Cache - batteries/0.55.1#ea02793f9c1e7b3bc24b7ff99a9a0a34 - Cache - boost/1.83.0#fd341e07fe362c605d187ed592f2fce8 - Cache - bzip2/1.0.8#d00dac990f08d991998d624be81a9526 - Cache - cli11/2.3.2#1424b9b1d9e3682a7122f415b078b4d7 - Cache - gflags/2.2.2#48d1262ffac8d30c3224befb8275a533 - Cache - glog/0.6.0#f25568477a1100349c4c2959ce946717 - Cache - gtest/1.14.0#25e2a474b4d1aecf5ff4f0555dcdf72c - Cache - libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1 - Cache - libfuse/3.16.2#b426f95ed62675cf8fb54bc407101696 - Cache - libunwind/1.7.2#4f36792edb3ed2c6c6525397f5643c66 - Cache - liburing/2.4#463d05ef1b3465d36eba39a7f2844df3 - Cache - openssl/3.2.0#d52f6cbf28a54701393831929f47dbbd - Cache - pcg-cpp/cci.20220409#b3c4e07c6ef6c2b5ddf2886f8a637bc7 - Cache - protobuf/5.27.0#ccce9aa25886556c6d66c77b2be4d806 - Cache - zlib/1.3#5c0f3a1a222eebb6bff34980bcd3e024 - Cache -Test requirements - gtest/cci.20210126#dafbdf84b58cd687075ace7314651c1a - Cache -Build requirements - b2/5.2.1#91bc73931a0acb655947a81569ed8b80 - Cache - cmake/3.26.4#0b40747e190e755932767f2ec4768ff5 - Cache - cmake/3.30.1#6d832cf2d46f6ec969ca5ed5b41f91eb - Cache - meson/1.2.0#68ba3b73f547af9c46816b2ef842a99d - Cache - ninja/1.12.1#fd583651bf0c6a901943495d49878803 - Cache -Resolved version ranges - b2/[>=5.2 <6]: b2/5.2.1 - cmake/[>=3.16 <4]: cmake/3.30.1 - gtest/[>=1.14.0]: gtest/cci.20210126 - ninja/[>=1.10.2 <2]: ninja/1.12.1 - pcg-cpp/[>=cci.20220409]: pcg-cpp/cci.20220409 - protobuf/[>=3.21.12]: protobuf/5.27.0 -Overrides - zlib/[>=1.2.11 <2]: ['zlib/1.3'] - libunwind/1.8.0: ['libunwind/1.7.2'] - -======== Computing necessary packages ======== -abseil/20240116.2: Checking 7 compatible configurations -abseil/20240116.2: Main binary package '427ba685c637d4b02789cda2bd590c754eb03890' missing. Using compatible package 'e951c2cb74ff982f51710131ee36b3e28f8e4563': compiler.cppstd=17 -gflags/2.2.2: Checking 11 compatible configurations -gflags/2.2.2: Main binary package '44eabc4771aeee384507fc08c863c6435178263c' missing. Using compatible package '3d216c4a794ad4ead4d22073c6f5e44018989263': compiler.cppstd=17 -gtest/1.14.0: Checking 7 compatible configurations -gtest/1.14.0: Main binary package 'dec6797a53c8ee92bc1c6586540cd6f2972efe5b' missing. Using compatible package '8c79091f8481612ee4242f0f3ef3cc13a5a12800': compiler.cppstd=17 -gtest/cci.20210126: Checking 9 compatible configurations -gtest/cci.20210126: Main binary package 'f4512038026b92eb5f53a1892437be6d7a527c40' missing. Using compatible package 'ad418b1d6effc6877f158337ee28e1f5cb0fc13a': compiler.cppstd=17 -protobuf/5.27.0: Checking 7 compatible configurations -protobuf/5.27.0: Main binary package 'fe688174d9d3f9d41ff7419b12114d243c3f8481' missing. Using compatible package '5fe12e7b087c0c4e0ee89ed683371984f642bce4': compiler.cppstd=17 -glog/0.6.0: Checking 11 compatible configurations -glog/0.6.0: Main binary package '00615d8b4cc96735b0723ca020d03b8b83bd7540' missing. Using compatible package 'b87780c0fc63b82ede7a99db267fd11711444420': compiler.cppstd=17 -batteries/0.55.1: Checking 11 compatible configurations -batteries/0.55.1: Main binary package '9d861a1e8ed014810eb7ff93c7bc5679e59011bd' missing. Using compatible package 'd77d782ece1b8314e57db1b08eb99bf84b8ced68': compiler.cppstd=17 -Requirements - abseil/20240116.2#54f81a20ccd26a6558e18d57059847e2:e951c2cb74ff982f51710131ee36b3e28f8e4563#00d86a5fbcd989bb64c057f821f767a8 - Cache - batteries/0.55.1#ea02793f9c1e7b3bc24b7ff99a9a0a34:d77d782ece1b8314e57db1b08eb99bf84b8ced68#e21b5a821423d65308ac97cf15f3501a - Cache - boost/1.83.0#fd341e07fe362c605d187ed592f2fce8:f7b96315a752d5ea8ef3c2eec56915a9bd765c81#9c7df5af22e95b30dc8744ea52d5d940 - Cache - bzip2/1.0.8#d00dac990f08d991998d624be81a9526:d2fd0bf682562652eedb970d66e83aded80b3fb1#6cf5c17b419f33a0a925d1d8e281d3cd - Cache - cli11/2.3.2#1424b9b1d9e3682a7122f415b078b4d7:da39a3ee5e6b4b0d3255bfef95601890afd80709#e04bf357d9bd58bd3f1042ff121bf921 - Cache - gflags/2.2.2#48d1262ffac8d30c3224befb8275a533:3d216c4a794ad4ead4d22073c6f5e44018989263#908a0eb0138f96b6dc4ed65828b96779 - Cache - glog/0.6.0#f25568477a1100349c4c2959ce946717:b87780c0fc63b82ede7a99db267fd11711444420#207682f0e58e96330acab25ad6d79832 - Cache - gtest/1.14.0#25e2a474b4d1aecf5ff4f0555dcdf72c:8c79091f8481612ee4242f0f3ef3cc13a5a12800#c3336292a69ce58cecd28ce76179735c - Cache - libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1:7141b46dc36f162c1999825c09b1ec68c17d6616#6e39386f8f08273ee4c765ac49c8dde2 - Cache - libfuse/3.16.2#b426f95ed62675cf8fb54bc407101696:7141b46dc36f162c1999825c09b1ec68c17d6616#1422880a721df66ec7d9a88e9f180972 - Cache - libunwind/1.7.2#4f36792edb3ed2c6c6525397f5643c66:76ad123c0b4672150480ad00bc8c5ebde5faacd0#bffcb69ae1800b801f12ab5f4201ad27 - Cache - liburing/2.4#463d05ef1b3465d36eba39a7f2844df3:893e43a1ce9cc27ea3549b60ceba7dccc92f04f9#4d36a32e2db28cd63a6f458013bf344a - Cache - openssl/3.2.0#d52f6cbf28a54701393831929f47dbbd:f7124f4882e8a147ba488837fa6a04d4bb37cbf7#f8082f47bb78718fa349b20379316359 - Cache - pcg-cpp/cci.20220409#b3c4e07c6ef6c2b5ddf2886f8a637bc7:da39a3ee5e6b4b0d3255bfef95601890afd80709#4703807cd2cb00aba979bcd262eff030 - Cache - protobuf/5.27.0#ccce9aa25886556c6d66c77b2be4d806:5fe12e7b087c0c4e0ee89ed683371984f642bce4#d806df8d48a82f04534cf590d741597c - Cache - zlib/1.3#5c0f3a1a222eebb6bff34980bcd3e024:7141b46dc36f162c1999825c09b1ec68c17d6616#3e28b9b20446b1f05240c36c569bb31d - Cache -Test requirements -Build requirements - cmake/3.26.4#0b40747e190e755932767f2ec4768ff5:63fead0844576fc02943e16909f08fcdddd6f44b#0e341efd7f3d724d3a793aacff5b86a3 - Cache -Skipped binaries - gtest/cci.20210126, b2/5.2.1, cmake/3.30.1, meson/1.2.0, ninja/1.12.1 - -======== Installing packages ======== - -======== Installing packages ======== -abseil/20240116.2: Already installed! (1 of 17) -bzip2/1.0.8: Already installed! (2 of 17) -cli11/2.3.2: Already installed! (3 of 17) -cmake/3.26.4: Already installed! (4 of 17) -cmake/3.26.4: Appending PATH environment variable: /ssh:a8-build-host:/home/gbornste/.a8.conan/.conan_2.0.16/p/cmake5aa963403baef/p/bin -gflags/2.2.2: Already installed! (5 of 17) -gtest/1.14.0: Already installed! (6 of 17) -libbacktrace/cci.20210118: Already installed! (7 of 17) -liburing/2.4: Already installed! (8 of 17) -pcg-cpp/cci.20220409: Already installed! (9 of 17) -zlib/1.3: Already installed! (10 of 17) -libfuse/3.16.2: Already installed! (11 of 17) -boost/1.83.0: Already installed! (12 of 17) -libunwind/1.7.2: Already installed! (13 of 17) -openssl/3.2.0: Already installed! (14 of 17) -protobuf/5.27.0: Already installed! (15 of 17) -glog/0.6.0: Already installed! (16 of 17) -batteries/0.55.1: Already installed! (17 of 17) -WARN: deprecated: Usage of deprecated Conan 1.X features that will be removed in Conan 2.X: -WARN: deprecated: 'cpp_info.names' used in: batteries/0.55.1, protobuf/5.27.0, zlib/1.3, boost/1.83.0, gtest/1.14.0, abseil/20240116.2, openssl/3.2.0, bzip2/1.0.8, cli11/2.3.2 -WARN: deprecated: 'cpp_info.build_modules' used in: protobuf/5.27.0, gflags/2.2.2, abseil/20240116.2, openssl/3.2.0, bzip2/1.0.8 -WARN: deprecated: 'env_info' used in: protobuf/5.27.0, boost/1.83.0, cmake/3.26.4, openssl/3.2.0, bzip2/1.0.8 -WARN: deprecated: 'cpp_info.filenames' used in: protobuf/5.27.0, boost/1.83.0 -WARN: deprecated: 'user_info' used in: boost/1.83.0 - -======== Finalizing install (deploy, generators) ======== -conanfile.py (llfs/0.1.0-devel): Calling generate() -conanfile.py (llfs/0.1.0-devel): Generators folder: /ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/build/RelWithDebInfo/generators -conanfile.py (llfs/0.1.0-devel): CMakeToolchain generated: conan_toolchain.cmake -conanfile.py (llfs/0.1.0-devel): Preset 'conan-relwithdebinfo' added to CMakePresets.json. Invoke it manually using 'cmake --preset conan-relwithdebinfo' if using CMake>=3.23 -conanfile.py (llfs/0.1.0-devel): If your CMake version is not compatible with CMakePresets (<3.23) call cmake like: 'cmake -G Ninja -DCMAKE_TOOLCHAIN_FILE=/ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/build/RelWithDebInfo/generators/conan_toolchain.cmake -DCMAKE_POLICY_DEFAULT_CMP0091=NEW -DCMAKE_BUILD_TYPE=RelWithDebInfo' -conanfile.py (llfs/0.1.0-devel): CMakeToolchain generated: CMakePresets.json -conanfile.py (llfs/0.1.0-devel): CMakeToolchain generated: ../../../src/CMakeUserPresets.json -conanfile.py (llfs/0.1.0-devel): CMakeDeps necessary find_package() and targets for your CMakeLists.txt - find_package(batteries) - find_package(Boost) - find_package(CLI11) - find_package(glog) - find_package(GTest) - find_package(libbacktrace) - find_package(OpenSSL) - find_package(liburing) - find_package(libfuse) - target_link_libraries(... batteries::batteries boost::boost CLI11::CLI11 glog::glog gtest::gtest libbacktrace::libbacktrace openssl::openssl liburing::liburing libfuse::libfuse) - -======== Generating conan_find_requirements.cmake ======== - -batteries/0.55.1, Traits: build=False, headers=True, libs=True, run=False, visible=True -boost/1.83.0, Traits: build=False, headers=True, libs=True, run=False, visible=True -cli11/2.3.2, Traits: build=False, headers=True, libs=False, run=False, visible=True -glog/0.6.0, Traits: build=False, headers=True, libs=True, run=False, visible=True -gtest/1.14.0, Traits: build=False, headers=True, libs=True, run=False, visible=True -libbacktrace/cci.20210118, Traits: build=False, headers=True, libs=True, run=False, visible=True -openssl/3.2.0, Traits: build=False, headers=True, libs=True, run=False, visible=True -zlib/1.3, Traits: build=False, headers=True, libs=True, run=False, visible=True -... skipping direct=False requirement -liburing/2.4, Traits: build=False, headers=True, libs=True, run=False, visible=True -libfuse/3.16.2, Traits: build=False, headers=True, libs=True, run=False, visible=True -libunwind/1.7.2, Traits: build=False, headers=True, libs=True, run=False, visible=True -... skipping direct=False requirement -cmake/3.26.4, Traits: build=True, headers=False, libs=False, run=True, visible=False -... skipping build=True requirement - -Saved file /ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/build/RelWithDebInfo/conan_find_requirements.cmake - -conanfile.py (llfs/0.1.0-devel): Generating aggregated env files -conanfile.py (llfs/0.1.0-devel): Generated aggregated env files: ['conanbuild.sh', 'conanrun.sh'] - -======== Calling build() ======== -conanfile.py (llfs/0.1.0-devel): Calling build() -conanfile.py (llfs/0.1.0-devel): Running CMake.configure() -conanfile.py (llfs/0.1.0-devel): RUN: cmake -G "Ninja" -DCMAKE_TOOLCHAIN_FILE="/ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/build/RelWithDebInfo/generators/conan_toolchain.cmake" -DCMAKE_INSTALL_PREFIX="/ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs" -DCMAKE_POLICY_DEFAULT_CMP0091="NEW" -DCMAKE_BUILD_TYPE="RelWithDebInfo" "/ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/src" --- Using Conan toolchain: /ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/build/RelWithDebInfo/generators/conan_toolchain.cmake --- Conan toolchain: C++ Standard 20 with extensions OFF --- Conan toolchain: Setting BUILD_SHARED_LIBS = OFF --- The CXX compiler identification is GNU 11.4.0 --- Detecting CXX compiler ABI info --- Detecting CXX compiler ABI info - done --- Check for working CXX compiler: /usr/bin/c++ - skipped --- Detecting CXX compile features --- Detecting CXX compile features - done -Enabled ccache builds --- Conan: Target declared 'batteries::batteries' --- Conan: Target declared 'pcg-cpp::pcg-cpp' --- Conan: Component target declared 'utf8_range::utf8_range' --- Conan: Component target declared 'utf8_range::utf8_validity' --- Conan: Component target declared 'protobuf::libprotobuf' --- Conan: Component target declared 'protobuf::libprotoc' --- Conan: Target declared 'protobuf::protobuf' --- Conan: Component target declared 'absl::config' --- Conan: Component target declared 'absl::dynamic_annotations' --- Conan: Component target declared 'absl::core_headers' --- Conan: Component target declared 'absl::pretty_function' --- Conan: Component target declared 'absl::fast_type_id' --- Conan: Component target declared 'absl::prefetch' --- Conan: Component target declared 'absl::algorithm' --- Conan: Component target declared 'absl::hashtable_debug_hooks' --- Conan: Component target declared 'absl::node_slot_policy' --- Conan: Component target declared 'absl::non_temporal_arm_intrinsics' --- Conan: Component target declared 'absl::non_temporal_memcpy' --- Conan: Component target declared 'absl::leak_check' --- Conan: Component target declared 'absl::flags_commandlineflag_internal' --- Conan: Component target declared 'absl::log_internal_config' --- Conan: Component target declared 'absl::log_internal_nullguard' --- Conan: Component target declared 'absl::log_internal_voidify' --- Conan: Component target declared 'absl::type_traits' --- Conan: Component target declared 'absl::meta' --- Conan: Component target declared 'absl::bits' --- Conan: Component target declared 'absl::int128' --- Conan: Component target declared 'absl::numeric' --- Conan: Component target declared 'absl::numeric_representation' --- Conan: Component target declared 'absl::exponential_biased' --- Conan: Component target declared 'absl::periodic_sampler' --- Conan: Component target declared 'absl::random_seed_gen_exception' --- Conan: Component target declared 'absl::random_internal_traits' --- Conan: Component target declared 'absl::random_internal_fast_uniform_bits' --- Conan: Component target declared 'absl::random_internal_iostream_state_saver' --- Conan: Component target declared 'absl::random_internal_wide_multiply' --- Conan: Component target declared 'absl::random_internal_fastmath' --- Conan: Component target declared 'absl::random_internal_pcg_engine' --- Conan: Component target declared 'absl::random_internal_platform' --- Conan: Component target declared 'absl::random_internal_randen_slow' --- Conan: Component target declared 'absl::random_internal_randen_hwaes_impl' --- Conan: Component target declared 'absl::random_internal_uniform_helper' --- Conan: Component target declared 'absl::has_ostream_operator' --- Conan: Component target declared 'absl::cordz_update_tracker' --- Conan: Component target declared 'absl::civil_time' --- Conan: Component target declared 'absl::time_zone' --- Conan: Component target declared 'absl::compare' --- Conan: Component target declared 'absl::if_constexpr' --- Conan: Component target declared 'absl::atomic_hook' --- Conan: Component target declared 'absl::errno_saver' --- Conan: Component target declared 'absl::log_severity' --- Conan: Component target declared 'absl::no_destructor' --- Conan: Component target declared 'absl::nullability' --- Conan: Component target declared 'absl::raw_logging_internal' --- Conan: Component target declared 'absl::base_internal' --- Conan: Component target declared 'absl::throw_delegate' --- Conan: Component target declared 'absl::scoped_set_env' --- Conan: Component target declared 'absl::strerror' --- Conan: Component target declared 'absl::algorithm_container' --- Conan: Component target declared 'absl::common_policy_traits' --- Conan: Component target declared 'absl::hashtable_debug' --- Conan: Component target declared 'absl::container_common' --- Conan: Component target declared 'absl::debugging_internal' --- Conan: Component target declared 'absl::overload' --- Conan: Component target declared 'absl::memory' --- Conan: Component target declared 'absl::random_internal_generate_real' --- Conan: Component target declared 'absl::random_internal_randen_hwaes' --- Conan: Component target declared 'absl::cordz_functions' --- Conan: Component target declared 'absl::bad_any_cast_impl' --- Conan: Component target declared 'absl::span' --- Conan: Component target declared 'absl::bad_optional_access' --- Conan: Component target declared 'absl::bad_variant_access' --- Conan: Component target declared 'absl::utility' --- Conan: Component target declared 'absl::spinlock_wait' --- Conan: Component target declared 'absl::base' --- Conan: Component target declared 'absl::endian' --- Conan: Component target declared 'absl::cleanup_internal' --- Conan: Component target declared 'absl::cleanup' --- Conan: Component target declared 'absl::compressed_tuple' --- Conan: Component target declared 'absl::fixed_array' --- Conan: Component target declared 'absl::inlined_vector_internal' --- Conan: Component target declared 'absl::inlined_vector' --- Conan: Component target declared 'absl::container_memory' --- Conan: Component target declared 'absl::hash_policy_traits' --- Conan: Component target declared 'absl::crc_cpu_detect' --- Conan: Component target declared 'absl::crc_internal' --- Conan: Component target declared 'absl::stacktrace' --- Conan: Component target declared 'absl::demangle_internal' --- Conan: Component target declared 'absl::debugging' --- Conan: Component target declared 'absl::any_invocable' --- Conan: Component target declared 'absl::bind_front' --- Conan: Component target declared 'absl::function_ref' --- Conan: Component target declared 'absl::city' --- Conan: Component target declared 'absl::low_level_hash' --- Conan: Component target declared 'absl::log_internal_conditions' --- Conan: Component target declared 'absl::random_internal_distribution_caller' --- Conan: Component target declared 'absl::random_internal_randen' --- Conan: Component target declared 'absl::string_view' --- Conan: Component target declared 'absl::charset' --- Conan: Component target declared 'absl::strings_internal' --- Conan: Component target declared 'absl::bad_any_cast' --- Conan: Component target declared 'absl::optional' --- Conan: Component target declared 'absl::variant' --- Conan: Component target declared 'absl::malloc_internal' --- Conan: Component target declared 'absl::random_bit_gen_ref' --- Conan: Component target declared 'absl::random_internal_mock_helpers' --- Conan: Component target declared 'absl::random_internal_randen_engine' --- Conan: Component target declared 'absl::strings' --- Conan: Component target declared 'absl::str_format_internal' --- Conan: Component target declared 'absl::graphcycles_internal' --- Conan: Component target declared 'absl::time' --- Conan: Component target declared 'absl::any' --- Conan: Component target declared 'absl::layout' --- Conan: Component target declared 'absl::symbolize' --- Conan: Component target declared 'absl::examine_stack' --- Conan: Component target declared 'absl::failure_signal_handler' --- Conan: Component target declared 'absl::flags_path_util' --- Conan: Component target declared 'absl::flags_commandlineflag' --- Conan: Component target declared 'absl::flags_private_handle_accessor' --- Conan: Component target declared 'absl::hash' --- Conan: Component target declared 'absl::log_internal_globals' --- Conan: Component target declared 'absl::log_internal_proto' --- Conan: Component target declared 'absl::log_internal_nullstream' --- Conan: Component target declared 'absl::log_internal_append_truncated' --- Conan: Component target declared 'absl::log_entry' --- Conan: Component target declared 'absl::log_sink' --- Conan: Component target declared 'absl::log_internal_fnmatch' --- Conan: Component target declared 'absl::random_distributions' --- Conan: Component target declared 'absl::random_internal_seed_material' --- Conan: Component target declared 'absl::random_internal_pool_urbg' --- Conan: Component target declared 'absl::random_internal_salted_seed_seq' --- Conan: Component target declared 'absl::random_internal_nonsecure_base' --- Conan: Component target declared 'absl::str_format' --- Conan: Component target declared 'absl::kernel_timeout_internal' --- Conan: Component target declared 'absl::synchronization' --- Conan: Component target declared 'absl::crc32c' --- Conan: Component target declared 'absl::crc_cord_state' --- Conan: Component target declared 'absl::flags_program_name' --- Conan: Component target declared 'absl::flags_config' --- Conan: Component target declared 'absl::flags_marshalling' --- Conan: Component target declared 'absl::flags_internal' --- Conan: Component target declared 'absl::log_internal_format' --- Conan: Component target declared 'absl::vlog_config_internal' --- Conan: Component target declared 'absl::absl_vlog_is_on' --- Conan: Component target declared 'absl::vlog_is_on' --- Conan: Component target declared 'absl::sample_recorder' --- Conan: Component target declared 'absl::random_seed_sequences' --- Conan: Component target declared 'absl::random_internal_distribution_test_util' --- Conan: Component target declared 'absl::cord_internal' --- Conan: Component target declared 'absl::cordz_statistics' --- Conan: Component target declared 'absl::cordz_handle' --- Conan: Component target declared 'absl::cordz_info' --- Conan: Component target declared 'absl::cordz_sample_token' --- Conan: Component target declared 'absl::cordz_update_scope' --- Conan: Component target declared 'absl::cord' --- Conan: Component target declared 'absl::btree' --- Conan: Component target declared 'absl::hash_function_defaults' --- Conan: Component target declared 'absl::hashtablez_sampler' --- Conan: Component target declared 'absl::raw_hash_set' --- Conan: Component target declared 'absl::log_globals' --- Conan: Component target declared 'absl::log_initialize' --- Conan: Component target declared 'absl::random_random' --- Conan: Component target declared 'absl::status' --- Conan: Component target declared 'absl::statusor' --- Conan: Component target declared 'absl::flat_hash_set' --- Conan: Component target declared 'absl::node_hash_set' --- Conan: Component target declared 'absl::raw_hash_map' --- Conan: Component target declared 'absl::log_internal_log_sink_set' --- Conan: Component target declared 'absl::log_sink_registry' --- Conan: Component target declared 'absl::flat_hash_map' --- Conan: Component target declared 'absl::node_hash_map' --- Conan: Component target declared 'absl::flags_reflection' --- Conan: Component target declared 'absl::flags' --- Conan: Component target declared 'absl::flags_usage_internal' --- Conan: Component target declared 'absl::flags_usage' --- Conan: Component target declared 'absl::flags_parse' --- Conan: Component target declared 'absl::log_internal_flags' --- Conan: Component target declared 'absl::log_internal_message' --- Conan: Component target declared 'absl::log_internal_strip' --- Conan: Component target declared 'absl::log_flags' --- Conan: Component target declared 'absl::log_internal_structured' --- Conan: Component target declared 'absl::log_structured' --- Conan: Component target declared 'absl::log_internal_check_op' --- Conan: Component target declared 'absl::log_internal_log_impl' --- Conan: Component target declared 'absl::absl_log' --- Conan: Component target declared 'absl::log' --- Conan: Component target declared 'absl::log_streamer' --- Conan: Component target declared 'absl::log_internal_check_impl' --- Conan: Component target declared 'absl::absl_check' --- Conan: Component target declared 'absl::check' --- Conan: Component target declared 'absl::die_if_null' --- Conan: Target declared 'abseil::abseil' --- Conan: Including build module from '/ssh:a8-build-host:/home/gbornste/.a8.conan/.conan_2.0.16/p/b/abseib7b4792619b83/p/lib/cmake/conan_trick/cxx_std.cmake' --- Conan: Target declared 'ZLIB::ZLIB' --- Conan: Including build module from '/ssh:a8-build-host:/home/gbornste/.a8.conan/.conan_2.0.16/p/b/protoffae71f0f7658/p/lib/cmake/protobuf/protobuf-generate.cmake' --- Conan: Including build module from '/ssh:a8-build-host:/home/gbornste/.a8.conan/.conan_2.0.16/p/b/protoffae71f0f7658/p/lib/cmake/protobuf/protobuf-module.cmake' --- Conan: Including build module from '/ssh:a8-build-host:/home/gbornste/.a8.conan/.conan_2.0.16/p/b/protoffae71f0f7658/p/lib/cmake/protobuf/protobuf-options.cmake' --- Conan: Including build module from '/ssh:a8-build-host:/home/gbornste/.a8.conan/.conan_2.0.16/p/b/protoffae71f0f7658/p/lib/cmake/protobuf/protobuf-conan-protoc-target.cmake' --- Conan: Component target declared 'Boost::diagnostic_definitions' --- Conan: Component target declared 'Boost::disable_autolinking' --- Conan: Component target declared 'Boost::dynamic_linking' --- Conan: Component target declared 'Boost::headers' --- Conan: Component target declared 'Boost::boost' --- Conan: Component target declared 'boost::_libboost' --- Conan: Component target declared 'Boost::atomic' --- Conan: Component target declared 'Boost::container' --- Conan: Component target declared 'Boost::context' --- Conan: Component target declared 'Boost::date_time' --- Conan: Component target declared 'Boost::exception' --- Conan: Component target declared 'Boost::math' --- Conan: Component target declared 'Boost::math_c99' --- Conan: Component target declared 'Boost::math_c99f' --- Conan: Component target declared 'Boost::math_c99l' --- Conan: Component target declared 'Boost::math_tr1' --- Conan: Component target declared 'Boost::math_tr1f' --- Conan: Component target declared 'Boost::math_tr1l' --- Conan: Component target declared 'Boost::program_options' --- Conan: Component target declared 'Boost::regex' --- Conan: Component target declared 'Boost::serialization' --- Conan: Component target declared 'Boost::stacktrace' --- Conan: Component target declared 'Boost::stacktrace_addr2line' --- Conan: Component target declared 'Boost::stacktrace_backtrace' --- Conan: Component target declared 'Boost::stacktrace_basic' --- Conan: Component target declared 'Boost::stacktrace_noop' --- Conan: Component target declared 'Boost::system' --- Conan: Component target declared 'Boost::timer' --- Conan: Component target declared 'Boost::url' --- Conan: Component target declared 'Boost::wserialization' --- Conan: Component target declared 'Boost::chrono' --- Conan: Component target declared 'Boost::coroutine' --- Conan: Component target declared 'Boost::filesystem' --- Conan: Component target declared 'Boost::json' --- Conan: Component target declared 'Boost::nowide' --- Conan: Component target declared 'Boost::random' --- Conan: Component target declared 'Boost::thread' --- Conan: Component target declared 'Boost::type_erasure' --- Conan: Component target declared 'Boost::wave' --- Conan: Component target declared 'Boost::contract' --- Conan: Component target declared 'Boost::fiber' --- Conan: Component target declared 'Boost::fiber_numa' --- Conan: Component target declared 'Boost::graph' --- Conan: Component target declared 'Boost::iostreams' --- Conan: Component target declared 'Boost::locale' --- Conan: Component target declared 'Boost::log' --- Conan: Component target declared 'Boost::log_setup' --- Conan: Target declared 'boost::boost' --- Conan: Target declared 'BZip2::BZip2' --- Conan: Including build module from '/ssh:a8-build-host:/home/gbornste/.a8.conan/.conan_2.0.16/p/b/bzip29e24ccc6c8ead/p/lib/cmake/conan-official-bzip2-variables.cmake' --- Conan: Target declared 'libbacktrace::libbacktrace' --- Conan: Target declared 'glog::glog' --- Conan: Target declared 'gflags::gflags' --- Conan: Component target declared 'libunwind::unwind' --- Conan: Component target declared 'libunwind::generic' --- Conan: Component target declared 'libunwind::ptrace' --- Conan: Component target declared 'libunwind::setjmp' --- Conan: Component target declared 'libunwind::coredump' --- Conan: Target declared 'libunwind::libunwind' --- Conan: Target declared 'CLI11::CLI11' --- Conan: Component target declared 'GTest::gtest' --- Conan: Component target declared 'GTest::gtest_main' --- Conan: Component target declared 'GTest::gmock' --- Conan: Component target declared 'GTest::gmock_main' --- Conan: Target declared 'gtest::gtest' --- Conan: Component target declared 'OpenSSL::Crypto' --- Conan: Component target declared 'OpenSSL::SSL' --- Conan: Target declared 'openssl::openssl' --- Conan: Including build module from '/ssh:a8-build-host:/home/gbornste/.a8.conan/.conan_2.0.16/p/b/opensdc5d99b668034/p/lib/cmake/conan-official-openssl-variables.cmake' --- Conan: Target declared 'liburing::liburing' --- Conan: Target declared 'libfuse::libfuse' --- Configuring done (0.3s) --- Generating done (0.1s) --- Build files have been written to: /ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/build/RelWithDebInfo - -conanfile.py (llfs/0.1.0-devel): Running CMake.build() -conanfile.py (llfs/0.1.0-devel): RUN: cmake --build "/ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/build/RelWithDebInfo" -- -j32 - [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/appendable_job.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/bloom_filter.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/buffered_log_data_reader.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/committable_page_cache_job.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/crc.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/data_layout.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/data_packer.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/data_packer_arena.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/data_reader.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/file_log_driver.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/file_log_driver/active_file.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/file_log_driver/concurrent_shared_state.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/file_log_driver/config.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/file_log_driver/flush_task_main.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/file_log_driver/location.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/file_log_driver/segment_file.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/file_segment_ref.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/filesystem.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/filesystem_page_device.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/finalized_job_tracker.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/finalized_page_cache_job.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/fuse.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring_buffer_pool.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring_buffer_view.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring_file.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring_file_runtime_options.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring_impl.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring_log_config2.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring_log_device2.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring_log_device2_metrics.cpp.o [0/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring_log_device_storage.cpp.o [1/200] Building CXX object CMakeFiles/llfs.dir/llfs/crc.cpp.o [1/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring_page_file_device.cpp.o [2/200] Building CXX object CMakeFiles/llfs.dir/llfs/data_layout.cpp.o [2/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring_stream_buffer.cpp.o [3/200] Building CXX object CMakeFiles/llfs.dir/llfs/data_reader.cpp.o [3/200] Building CXX object CMakeFiles/llfs.dir/llfs/latching_bit_set.cpp.o [4/200] Building CXX object CMakeFiles/llfs.dir/llfs/filesystem.cpp.o [4/200] Building CXX object CMakeFiles/llfs.dir/llfs/log_device_config2.cpp.o [5/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring_impl.cpp.o [5/200] Building CXX object CMakeFiles/llfs.dir/llfs/log_device_snapshot.cpp.o [6/200] Building CXX object CMakeFiles/llfs.dir/llfs/bloom_filter.cpp.o [6/200] Building CXX object CMakeFiles/llfs.dir/llfs/lru_clock.cpp.o [7/200] Building CXX object CMakeFiles/llfs.dir/llfs/data_packer_arena.cpp.o [7/200] Building CXX object CMakeFiles/llfs.dir/llfs/mem_fuse.cpp.o [8/200] Building CXX object CMakeFiles/llfs.dir/llfs/file_segment_ref.cpp.o [8/200] Building CXX object CMakeFiles/llfs.dir/llfs/mem_inode.cpp.o [9/200] Building CXX object CMakeFiles/llfs.dir/llfs/buffered_log_data_reader.cpp.o [9/200] Building CXX object CMakeFiles/llfs.dir/llfs/memory_log_device.cpp.o [10/200] Building CXX object CMakeFiles/llfs.dir/llfs/file_log_driver/active_file.cpp.o [10/200] Building CXX object CMakeFiles/llfs.dir/llfs/memory_page_arena.cpp.o [11/200] Building CXX object CMakeFiles/llfs.dir/llfs/file_log_driver/concurrent_shared_state.cpp.o [11/200] Building CXX object CMakeFiles/llfs.dir/llfs/memory_page_cache.cpp.o [12/200] Building CXX object CMakeFiles/llfs.dir/llfs/file_log_driver/segment_file.cpp.o [12/200] Building CXX object CMakeFiles/llfs.dir/llfs/memory_page_device.cpp.o [13/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring_buffer_view.cpp.o [13/200] Building CXX object CMakeFiles/llfs.dir/llfs/null_worker_task_fuse_impl.cpp.o [14/200] Building CXX object CMakeFiles/llfs.dir/llfs/data_packer.cpp.o [14/200] Building CXX object CMakeFiles/llfs.dir/llfs/opaque_page_view.cpp.o [15/200] Building CXX object CMakeFiles/llfs.dir/llfs/file_log_driver/config.cpp.o [15/200] Building CXX object CMakeFiles/llfs.dir/llfs/pack_as_raw.cpp.o [16/200] Building CXX object CMakeFiles/llfs.dir/llfs/file_log_driver/flush_task_main.cpp.o [16/200] Building CXX object CMakeFiles/llfs.dir/llfs/packable_ref.cpp.o [17/200] Building CXX object CMakeFiles/llfs.dir/llfs/file_log_driver/location.cpp.o [17/200] Building CXX object CMakeFiles/llfs.dir/llfs/packed_bytes.cpp.o [18/200] Building CXX object CMakeFiles/llfs.dir/llfs/filesystem_page_device.cpp.o [18/200] Building CXX object CMakeFiles/llfs.dir/llfs/packed_config.cpp.o [19/200] Building CXX object CMakeFiles/llfs.dir/llfs/finalized_job_tracker.cpp.o [19/200] Building CXX object CMakeFiles/llfs.dir/llfs/packed_float.cpp.o [20/200] Building CXX object CMakeFiles/llfs.dir/llfs/fuse.cpp.o [20/200] Building CXX object CMakeFiles/llfs.dir/llfs/packed_interval.cpp.o [21/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring.cpp.o [21/200] Building CXX object CMakeFiles/llfs.dir/llfs/packed_log_page_header.cpp.o [22/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring_buffer_pool.cpp.o [22/200] Building CXX object CMakeFiles/llfs.dir/llfs/packed_page_header.cpp.o [23/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring_file.cpp.o [23/200] Building CXX object CMakeFiles/llfs.dir/llfs/packed_page_id.cpp.o [24/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring_file_runtime_options.cpp.o [24/200] Building CXX object CMakeFiles/llfs.dir/llfs/packed_page_ref_count.cpp.o [25/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring_log_device_storage.cpp.o [25/200] Building CXX object CMakeFiles/llfs.dir/llfs/packed_slot_range.cpp.o [26/200] Building CXX object CMakeFiles/llfs.dir/llfs/appendable_job.cpp.o [26/200] Building CXX object CMakeFiles/llfs.dir/llfs/packed_status.cpp.o [27/200] Building CXX object CMakeFiles/llfs.dir/llfs/committable_page_cache_job.cpp.o [27/200] Building CXX object CMakeFiles/llfs.dir/llfs/packed_status_or.cpp.o [28/200] Building CXX object CMakeFiles/llfs.dir/llfs/file_log_driver.cpp.o [28/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_allocator.cpp.o [29/200] Building CXX object CMakeFiles/llfs.dir/llfs/finalized_page_cache_job.cpp.o [29/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_allocator_config.cpp.o [30/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring_log_config2.cpp.o [30/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_allocator_events.cpp.o [31/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring_log_device2.cpp.o [31/200] Building CXX object CMakeFiles/llfs.dir/llfs/volume_events.cpp.o [32/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring_log_device2_metrics.cpp.o [32/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_allocator_state.cpp.o [33/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring_page_file_device.cpp.o [33/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_allocator_state_no_lock.cpp.o [34/200] Building CXX object CMakeFiles/llfs.dir/llfs/ioring_stream_buffer.cpp.o [34/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_arena_config.cpp.o [35/200] Building CXX object CMakeFiles/llfs.dir/llfs/latching_bit_set.cpp.o [35/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_buffer.cpp.o [36/200] Building CXX object CMakeFiles/llfs.dir/llfs/log_device_snapshot.cpp.o [36/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_cache.cpp.o [37/200] Building CXX object CMakeFiles/llfs.dir/llfs/lru_clock.cpp.o [37/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_cache_config_file.cpp.o [38/200] Building CXX object CMakeFiles/llfs.dir/llfs/log_device_config2.cpp.o [38/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_cache_job.cpp.o [39/200] Building CXX object CMakeFiles/llfs.dir/llfs/mem_fuse.cpp.o [39/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_cache_options.cpp.o [40/200] Building CXX object CMakeFiles/llfs.dir/llfs/mem_inode.cpp.o [40/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_cache_slot.cpp.o [41/200] Building CXX object CMakeFiles/llfs.dir/llfs/memory_log_device.cpp.o [41/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_cache_slot_pool.cpp.o [42/200] Building CXX object CMakeFiles/llfs.dir/llfs/packed_config.cpp.o [42/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_device.cpp.o [43/200] Building CXX object CMakeFiles/llfs.dir/llfs/packed_float.cpp.o [43/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_device_cache.cpp.o [44/200] Building CXX object CMakeFiles/llfs.dir/llfs/memory_page_device.cpp.o [44/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_device_config.cpp.o [45/200] Building CXX object CMakeFiles/llfs.dir/llfs/null_worker_task_fuse_impl.cpp.o [45/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_filter.cpp.o [46/200] Building CXX object CMakeFiles/llfs.dir/llfs/packed_log_page_header.cpp.o [46/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_graph_node.cpp.o [47/200] Building CXX object CMakeFiles/llfs.dir/llfs/memory_page_arena.cpp.o [47/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_id_slot.cpp.o [48/200] Building CXX object CMakeFiles/llfs.dir/llfs/pack_as_raw.cpp.o [48/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_layout.cpp.o [49/200] Building CXX object CMakeFiles/llfs.dir/llfs/packable_ref.cpp.o [49/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_layout_id.cpp.o [50/200] Building CXX object CMakeFiles/llfs.dir/llfs/packed_bytes.cpp.o [50/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_loader.cpp.o [51/200] Building CXX object CMakeFiles/llfs.dir/llfs/packed_page_header.cpp.o [51/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_recycler.cpp.o [52/200] Building CXX object CMakeFiles/llfs.dir/llfs/packed_page_id.cpp.o [52/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_recycler_events.cpp.o [53/200] Building CXX object CMakeFiles/llfs.dir/llfs/memory_page_cache.cpp.o [53/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_recycler_options.cpp.o [54/200] Building CXX object CMakeFiles/llfs.dir/llfs/opaque_page_view.cpp.o [54/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_recycler_recovery_visitor.cpp.o [55/200] Building CXX object CMakeFiles/llfs.dir/llfs/packed_page_ref_count.cpp.o [55/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_recycler_state.cpp.o [56/200] Building CXX object CMakeFiles/llfs.dir/llfs/packed_interval.cpp.o [56/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_ref_count.cpp.o [57/200] Building CXX object CMakeFiles/llfs.dir/llfs/packed_status.cpp.o [57/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_view.cpp.o [58/200] Building CXX object CMakeFiles/llfs.dir/llfs/packed_slot_range.cpp.o [58/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_write_op.cpp.o [59/200] Building CXX object CMakeFiles/llfs.dir/llfs/packed_status_or.cpp.o [59/200] Building CXX object CMakeFiles/llfs.dir/llfs/pinned_page.cpp.o [60/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_cache_options.cpp.o [60/200] Building CXX object CMakeFiles/llfs.dir/llfs/raw_block_file.cpp.o [61/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_allocator_events.cpp.o [61/200] Building CXX object CMakeFiles/llfs.dir/llfs/raw_block_file_impl.cpp.o [62/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_layout_id.cpp.o [62/200] Building CXX object CMakeFiles/llfs.dir/llfs/raw_volume_log_data_parser.cpp.o [63/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_cache_config_file.cpp.o [63/200] Building CXX object CMakeFiles/llfs.dir/llfs/ring_buffer.cpp.o [64/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_buffer.cpp.o [64/200] Building CXX object CMakeFiles/llfs.dir/llfs/sha256.cpp.o [65/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_allocator_state_no_lock.cpp.o [65/200] Building CXX object CMakeFiles/llfs.dir/llfs/simulated_log_device.cpp.o [66/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_allocator_state.cpp.o [66/200] Building CXX object CMakeFiles/llfs.dir/llfs/simulated_log_device_impl.cpp.o [67/200] Building CXX object CMakeFiles/llfs.dir/llfs/volume_events.cpp.o [67/200] Building CXX object CMakeFiles/llfs.dir/llfs/simulated_log_device_reader_impl.cpp.o [68/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_device.cpp.o [68/200] Building CXX object CMakeFiles/llfs.dir/llfs/simulated_log_device_storage.cpp.o [69/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_allocator.cpp.o [69/200] Building CXX object CMakeFiles/llfs.dir/llfs/simulated_log_device_writer_impl.cpp.o [70/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_allocator_config.cpp.o [70/200] Building CXX object CMakeFiles/llfs.dir/llfs/simulated_page_device.cpp.o [71/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_ref_count.cpp.o [71/200] Building CXX object CMakeFiles/llfs.dir/llfs/simulated_page_device_impl.cpp.o [72/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_arena_config.cpp.o [72/200] Building CXX object CMakeFiles/llfs.dir/llfs/slot.cpp.o [73/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_cache_slot.cpp.o [73/200] Building CXX object CMakeFiles/llfs.dir/llfs/slot_interval_map.cpp.o [74/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_device_cache.cpp.o [74/200] Building CXX object CMakeFiles/llfs.dir/llfs/slot_lock_manager.cpp.o [75/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_layout.cpp.o [75/200] Building CXX object CMakeFiles/llfs.dir/llfs/slot_parse.cpp.o [76/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_cache_job.cpp.o [76/200] Building CXX object CMakeFiles/llfs.dir/llfs/slot_reader.cpp.o [77/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_filter.cpp.o [77/200] Building CXX object CMakeFiles/llfs.dir/llfs/slot_sequencer.cpp.o [78/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_id_slot.cpp.o [78/200] Building CXX object CMakeFiles/llfs.dir/llfs/slot_writer.cpp.o [79/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_recycler_events.cpp.o [79/200] Building CXX object CMakeFiles/llfs.dir/llfs/status_code.cpp.o [80/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_recycler_options.cpp.o [80/200] Building CXX object CMakeFiles/llfs.dir/llfs/storage_context.cpp.o [81/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_device_config.cpp.o [81/200] Building CXX object CMakeFiles/llfs.dir/llfs/storage_file.cpp.o [82/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_graph_node.cpp.o [82/200] Building CXX object CMakeFiles/llfs.dir/llfs/storage_file_builder.cpp.o [83/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_loader.cpp.o [83/200] Building CXX object CMakeFiles/llfs.dir/llfs/storage_file_config_block.cpp.o [84/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_view.cpp.o [84/200] Building CXX object CMakeFiles/llfs.dir/llfs/storage_object_info.cpp.o [85/200] Building CXX object CMakeFiles/llfs.dir/llfs/raw_block_file.cpp.o [85/200] Building CXX object CMakeFiles/llfs.dir/llfs/storage_simulation.cpp.o [86/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_recycler_recovery_visitor.cpp.o [86/200] Building CXX object CMakeFiles/llfs.dir/llfs/strings.cpp.o [87/200] Building CXX object CMakeFiles/llfs.dir/llfs/ring_buffer.cpp.o [87/200] Building CXX object CMakeFiles/llfs.dir/llfs/system_config.cpp.o [88/200] Building CXX object CMakeFiles/llfs.dir/llfs/sha256.cpp.o [88/200] Building CXX object CMakeFiles/llfs.dir/llfs/testing/fake_log_device.cpp.o [89/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_cache_slot_pool.cpp.o [89/200] Building CXX object CMakeFiles/llfs.dir/llfs/testing/fake_page_device.cpp.o [90/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_recycler_state.cpp.o [90/200] Building CXX object CMakeFiles/llfs.dir/llfs/testing/test_config.cpp.o [91/200] Building CXX object CMakeFiles/llfs.dir/llfs/pinned_page.cpp.o [91/200] Building CXX object CMakeFiles/llfs.dir/llfs/testing/util.cpp.o [92/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_recycler.cpp.o [92/200] Building CXX object CMakeFiles/llfs.dir/llfs/track_fds.cpp.o [93/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_write_op.cpp.o [93/200] Building CXX object CMakeFiles/llfs.dir/llfs/trie.cpp.o [94/200] Building CXX object CMakeFiles/llfs.dir/llfs/raw_block_file_impl.cpp.o [94/200] Building CXX object CMakeFiles/llfs.dir/llfs/user_data.cpp.o [95/200] Building CXX object CMakeFiles/llfs.dir/llfs/raw_volume_log_data_parser.cpp.o [95/200] Building CXX object CMakeFiles/llfs.dir/llfs/varint.cpp.o [96/200] Building CXX object CMakeFiles/llfs.dir/llfs/status_code.cpp.o [96/200] Building CXX object CMakeFiles/llfs.dir/llfs/volume.cpp.o [97/200] Building CXX object CMakeFiles/llfs.dir/llfs/slot_interval_map.cpp.o [97/200] Building CXX object CMakeFiles/llfs.dir/llfs/volume_config.cpp.o [98/200] Building CXX object CMakeFiles/llfs.dir/llfs/simulated_log_device.cpp.o [98/200] Building CXX object CMakeFiles/llfs.dir/llfs/volume_job_recovery_visitor.cpp.o [99/200] Building CXX object CMakeFiles/llfs.dir/llfs/simulated_log_device_impl.cpp.o [99/200] Building CXX object CMakeFiles/llfs.dir/llfs/volume_metadata_recovery_visitor.cpp.o [100/200] Building CXX object CMakeFiles/llfs.dir/llfs/slot.cpp.o [100/200] Building CXX object CMakeFiles/llfs.dir/llfs/volume_metadata_refresher.cpp.o [101/200] Building CXX object CMakeFiles/llfs.dir/llfs/slot_parse.cpp.o [101/200] Building CXX object CMakeFiles/llfs.dir/llfs/volume_reader.cpp.o [102/200] Building CXX object CMakeFiles/llfs.dir/llfs/system_config.cpp.o [102/200] Building CXX object CMakeFiles/llfs.dir/llfs/volume_runtime_options.cpp.o [103/200] Building CXX object CMakeFiles/llfs.dir/llfs/simulated_log_device_reader_impl.cpp.o [103/200] Building CXX object CMakeFiles/llfs.dir/llfs/volume_trimmed_region_visitor.cpp.o [104/200] Building CXX object CMakeFiles/llfs.dir/llfs/simulated_page_device.cpp.o [104/200] Building CXX object CMakeFiles/llfs.dir/llfs/volume_trimmer.cpp.o [105/200] Building CXX object CMakeFiles/llfs.dir/llfs/slot_lock_manager.cpp.o [105/200] Building CXX object CMakeFiles/llfs.dir/llfs/volume_trimmer_recovery_visitor.cpp.o [106/200] Building CXX object CMakeFiles/llfs.dir/llfs/slot_reader.cpp.o [106/200] Building CXX object CMakeFiles/llfs.dir/llfs/worker_task.cpp.o [107/200] Building CXX object CMakeFiles/llfs.dir/llfs/slot_sequencer.cpp.o [107/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/data_packer.test.cpp.o [108/200] Building CXX object CMakeFiles/llfs.dir/llfs/strings.cpp.o [108/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/filesystem_page_device.test.cpp.o [109/200] Building CXX object CMakeFiles/llfs.dir/llfs/testing/util.cpp.o [109/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/cache.test.cpp.o [110/200] Building CXX object CMakeFiles/llfs.dir/llfs/track_fds.cpp.o [110/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/buffered_log_data_reader.test.cpp.o [111/200] Building CXX object CMakeFiles/llfs.dir/llfs/simulated_log_device_storage.cpp.o [111/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/buffer.test.cpp.o [112/200] Building CXX object CMakeFiles/llfs.dir/llfs/simulated_log_device_writer_impl.cpp.o [112/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/bloom_filter.test.cpp.o [113/200] Building CXX object CMakeFiles/llfs.dir/llfs/slot_writer.cpp.o [113/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/file_log_driver.test.cpp.o [114/200] Building CXX object CMakeFiles/llfs.dir/llfs/storage_file.cpp.o [114/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/fuse.test.cpp.o [115/200] Building CXX object CMakeFiles/llfs.dir/llfs/storage_file_config_block.cpp.o [115/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/ioring.test.cpp.o [116/200] Building CXX object CMakeFiles/llfs.dir/llfs/storage_object_info.cpp.o [116/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/ioring_buffer_pool.test.cpp.o [117/200] Building CXX object CMakeFiles/llfs.dir/llfs/testing/fake_log_device.cpp.o [117/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/ioring_buffer_view.test.cpp.o [118/200] Building CXX object CMakeFiles/llfs.dir/llfs/testing/fake_page_device.cpp.o [118/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/ioring_file.test.cpp.o [119/200] Building CXX object CMakeFiles/llfs.dir/llfs/testing/test_config.cpp.o [119/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/ioring_log_device2.test.cpp.o [120/200] Building CXX object CMakeFiles/llfs.dir/llfs/user_data.cpp.o [120/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/ioring_stream_buffer.test.cpp.o [121/200] Building CXX object CMakeFiles/llfs.dir/llfs/varint.cpp.o [121/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/key.test.cpp.o [122/200] Building CXX object CMakeFiles/llfs.dir/llfs/simulated_page_device_impl.cpp.o [122/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/latching_bit_set.test.cpp.o [123/200] Building CXX object CMakeFiles/llfs.dir/llfs/storage_context.cpp.o [123/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/log_device.test.cpp.o [124/200] Building CXX object CMakeFiles/llfs.dir/llfs/storage_file_builder.cpp.o [124/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/lru_clock.test.cpp.o [125/200] Building CXX object CMakeFiles/llfs.dir/llfs/trie.cpp.o [125/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/mem_fuse.test.cpp.o [126/200] Building CXX object CMakeFiles/llfs.dir/llfs/storage_simulation.cpp.o [126/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/memory_page_device.test.cpp.o [127/200] Building CXX object CMakeFiles/llfs.dir/llfs/volume.cpp.o [127/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/pack_as_raw.test.cpp.o [128/200] Building CXX object CMakeFiles/llfs.dir/llfs/volume_metadata_recovery_visitor.cpp.o [128/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/packed_array.test.cpp.o [129/200] Building CXX object CMakeFiles/llfs.dir/llfs/volume_config.cpp.o [129/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/packed_bytes.test.cpp.o [130/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/buffer.test.cpp.o [130/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/packed_float.test.cpp.o [131/200] Building CXX object CMakeFiles/llfs.dir/llfs/volume_metadata_refresher.cpp.o [131/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/packed_interval.test.cpp.o [132/200] Building CXX object CMakeFiles/llfs.dir/llfs/volume_runtime_options.cpp.o [132/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/packed_slot_range.test.cpp.o [133/200] Building CXX object CMakeFiles/llfs.dir/llfs/volume_job_recovery_visitor.cpp.o [133/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/packed_status.test.cpp.o [134/200] Building CXX object CMakeFiles/llfs.dir/llfs/volume_reader.cpp.o [134/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/packed_status_or.test.cpp.o [135/200] Building CXX object CMakeFiles/llfs.dir/llfs/worker_task.cpp.o [135/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/packed_variant.test.cpp.o [136/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/data_packer.test.cpp.o [136/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/page_allocator.test.cpp.o [137/200] Building CXX object CMakeFiles/llfs.dir/llfs/volume_trimmed_region_visitor.cpp.o [137/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/page_cache_slot.test.cpp.o [138/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/buffered_log_data_reader.test.cpp.o [138/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/page_recycler.test.cpp.o [139/200] Building CXX object CMakeFiles/llfs.dir/llfs/volume_trimmer_recovery_visitor.cpp.o [139/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/raw_volume_log_data_parser.test.cpp.o [140/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/filesystem_page_device.test.cpp.o [140/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/ring_buffer.test.cpp.o [141/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/key.test.cpp.o [141/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/sha256.test.cpp.o [142/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/latching_bit_set.test.cpp.o [142/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/slot.test.cpp.o [143/200] Building CXX object CMakeFiles/llfs.dir/llfs/volume_trimmer.cpp.o [143/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/slot_interval_map.test.cpp.o [144/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/lru_clock.test.cpp.o [144/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/slot_lock_manager.test.cpp.o [145/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/cache.test.cpp.o [145/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/slot_read_lock.test.cpp.o [146/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/ioring_buffer_view.test.cpp.o [146/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/slot_sequencer.test.cpp.o [147/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/fuse.test.cpp.o [147/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/slot_test.cpp.o [148/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/bloom_filter.test.cpp.o [148/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/slot_writer.test.cpp.o [149/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/ioring_file.test.cpp.o [149/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/storage_context.test.cpp.o [150/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/ioring_stream_buffer.test.cpp.o [150/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/storage_file_builder.test.cpp.o [151/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/ioring.test.cpp.o [151/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/strings.test.cpp.o [152/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/ioring_buffer_pool.test.cpp.o [152/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/track_fds.test.cpp.o [153/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/packed_float.test.cpp.o [153/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/trie.test.cpp.o [154/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/file_log_driver.test.cpp.o [154/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/unpack_cast.test.cpp.o [155/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/log_device.test.cpp.o [155/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/varint.test.cpp.o [156/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/memory_page_device.test.cpp.o [156/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/volume.test.cpp.o [157/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/pack_as_raw.test.cpp.o [157/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/volume_config.test.cpp.o [158/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/mem_fuse.test.cpp.o [158/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/volume_events.test.cpp.o [159/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/packed_bytes.test.cpp.o [159/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/volume_multi_append.test.cpp.o [160/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/ioring_log_device2.test.cpp.o [160/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/volume_trimmer.test.cpp.o [161/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/packed_array.test.cpp.o [161/200] Building CXX object CMakeFiles/llfs_Test.dir/common/test_environment.cpp.o [162/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/packed_interval.test.cpp.o [162/200] Building CXX object CMakeFiles/llfs_cli.dir/llfs_cli/main.cpp.o [163/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/ring_buffer.test.cpp.o [163/200] Building CXX object CMakeFiles/llfs_cli.dir/llfs_cli/list_command.cpp.o [164/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/packed_slot_range.test.cpp.o [164/200] Building CXX object CMakeFiles/llfs_fuse.dir/llfs_fuse/main.cpp.o [165/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/packed_status.test.cpp.o [166/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/sha256.test.cpp.o [167/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/packed_status_or.test.cpp.o [168/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/packed_variant.test.cpp.o [169/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/slot.test.cpp.o [170/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/strings.test.cpp.o [171/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/slot_interval_map.test.cpp.o [172/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/page_cache_slot.test.cpp.o [173/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/slot_lock_manager.test.cpp.o [174/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/unpack_cast.test.cpp.o [175/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/slot_test.cpp.o [176/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/varint.test.cpp.o [177/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/slot_writer.test.cpp.o [178/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/track_fds.test.cpp.o [179/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/slot_sequencer.test.cpp.o [180/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/slot_read_lock.test.cpp.o [181/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/trie.test.cpp.o [182/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/raw_volume_log_data_parser.test.cpp.o [183/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/page_recycler.test.cpp.o [184/200] Building CXX object CMakeFiles/llfs_Test.dir/common/test_environment.cpp.o [185/200] Building CXX object CMakeFiles/llfs_cli.dir/llfs_cli/main.cpp.o [186/200] Building CXX object CMakeFiles/llfs_cli.dir/llfs_cli/list_command.cpp.o [187/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/volume_events.test.cpp.o [188/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/page_allocator.test.cpp.o [189/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/storage_file_builder.test.cpp.o [190/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/volume_multi_append.test.cpp.o [191/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/storage_context.test.cpp.o [192/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/volume_config.test.cpp.o [193/200] Building CXX object CMakeFiles/llfs_fuse.dir/llfs_fuse/main.cpp.o [194/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/volume_trimmer.test.cpp.o [195/200] Building CXX object CMakeFiles/llfs_Test.dir/llfs/volume.test.cpp.o [196/200] Building CXX object CMakeFiles/llfs.dir/llfs/page_cache.cpp.o [196/200] Linking CXX static library libllfs.a [197/200] Linking CXX static library libllfs.a [197/200] Linking CXX executable llfs [197/200] Linking CXX executable llfs_Test [197/200] Linking CXX executable llfs_fuse [198/200] Linking CXX executable llfs_fuse [199/200] Linking CXX executable llfs [200/200] Linking CXX executable llfs_Test - -"/ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/script/generate-vscode-config.sh" -vscode_config_dir is /ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/.vscode -project_dir is /ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs -checking for /ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/build/Release/compile_commands.json -checking for /ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/build/RelWithDebInfo/compile_commands.json -checking for /ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/build/Debug/compile_commands.json -(cd "/ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/build/RelWithDebInfo" && GTEST_OUTPUT='xml:../test-results.xml' ./llfs_Test) -Running main() from /ssh:a8-build-host:/home/gbornste/.a8.conan/.conan_2.0.16/p/b/gtest97ac6ae44faba/b/src/googletest/src/gtest_main.cc -Note: Google Test filter = StorageContextTest* -[==========] Running 3 tests from 1 test suite. -[----------] Global test environment set-up. -[----------] 3 tests from StorageContextTest -[ RUN ] StorageContextTest.GetPageCache -WARNING: Logging before InitGoogleLogging() is written to STDERR -I20240909 15:20:12.000912 567 page_cache.cpp:525] Are we calling add_page_devices twice? -I20240909 15:20:12.000965 567 page_cache.cpp:541] max_page_device_id found: 1 -I20240909 15:20:12.000975 567 page_cache.cpp:548] Arena has device_id: 1 -I20240909 15:20:12.001009 567 page_cache.cpp:548] Arena has device_id: 1 -E20240909 15:20:12.001017 567 page_cache.cpp:564] FATAL: /ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/src/llfs/page_cache.cpp:564: Assertion failed: state->page_devices[device_id] == nullptr - (in `batt::Status llfs::PageCache::add_page_devices(std::vector&)`) - - state->page_devices[device_id] == 0x5640536ca4f0 - - nullptr == nullptr - -Duplicate entries found for the same device id! device_id == 1 -E20240909 15:20:12.001060 567 assert_impl.hpp:14] - -FATAL: signal 6 (Aborted): -[[raw stack]] -./llfs_Test(+0xe878f)[0x564050b5978f] -/lib/x86_64-linux-gnu/libc.so.6(+0x45495)[0x7f1ab2e65495] -/lib/x86_64-linux-gnu/libc.so.6(on_exit+0x0)[0x7f1ab2e65610] -./llfs_Test(+0xdb465)[0x564050b4c465] -/lib/x86_64-linux-gnu/libc.so.6(+0x42520)[0x7f1ab2e62520] -/lib/x86_64-linux-gnu/libc.so.6(pthread_kill+0x12c)[0x7f1ab2eb69fc] -/lib/x86_64-linux-gnu/libc.so.6(raise+0x16)[0x7f1ab2e62476] -/lib/x86_64-linux-gnu/libc.so.6(abort+0xd3)[0x7f1ab2e487f3] -./llfs_Test(+0x566cec)[0x564050fd7cec] -./llfs_Test(+0x44f791)[0x564050ec0791] -./llfs_Test(+0x452649)[0x564050ec3649] -./llfs_Test(+0x452e63)[0x564050ec3e63] -./llfs_Test(+0x4ce936)[0x564050f3f936] -./llfs_Test(+0x2e4b58)[0x564050d55b58] -./llfs_Test(+0x715f97)[0x564051186f97] -./llfs_Test(+0x706d31)[0x564051177d31] -./llfs_Test(+0x706f75)[0x564051177f75] -./llfs_Test(+0x7094a1)[0x56405117a4a1] -./llfs_Test(+0x70d27a)[0x56405117e27a] -./llfs_Test(+0x71657f)[0x56405118757f] -./llfs_Test(+0x707035)[0x564051178035] -./llfs_Test(+0xd4747)[0x564050b45747] -/lib/x86_64-linux-gnu/libc.so.6(+0x29d90)[0x7f1ab2e49d90] -/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0x80)[0x7f1ab2e49e40] -./llfs_Test(+0xd4d95)[0x564050b45d95] - - 0# batt::print_stack_trace() at /ssh:a8-build-host:/home/gbornste/.a8.conan/.conan_2.0.16/p/b/batte897795ee12239/b/src/batteries/segv.hpp:54 - 1# __run_exit_handlers at ./stdlib/exit.c:114 - 2# on_exit at ./stdlib/on_exit.c:26 - 3# 0x0000564050B4C465 in ./llfs_Test - 4# 0x00007F1AB2E62520 in /lib/x86_64-linux-gnu/libc.so.6 - 5# pthread_kill at ./nptl/pthread_kill.c:89 - 6# raise at ../sysdeps/posix/raise.c:27 - 7# abort at ./stdlib/abort.c:81 - 8# batt::fail_check_exit() at /ssh:a8-build-host:/home/gbornste/.a8.conan/.conan_2.0.16/p/b/batte897795ee12239/b/src/batteries/assert_impl.hpp:15 - 9# llfs::PageCache::add_page_devices(std::vector >&) at /ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/src/llfs/page_cache.cpp:564 -10# llfs::PageCache::PageCache(std::vector >&&, llfs::PageCacheOptions const&) at /ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/src/llfs/page_cache.cpp:110 -11# llfs::PageCache::make_shared(std::vector >&&, llfs::PageCacheOptions const&) at /ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/src/llfs/page_cache.cpp:92 -12# llfs::StorageContext::get_page_cache() at /ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/src/llfs/storage_context.cpp:350 -13# (anonymous namespace)::StorageContextTest::SetUp() at /ssh:a8-build-host:/home/gbornste/Desktop/Projects/MPS/llfs/src/llfs/storage_context.test.cpp:132 -14# void testing::internal::HandleExceptionsInMethodIfSupported(testing::Test*, void (testing::Test::*)(), char const*) at /ssh:a8-build-host:/home/gbornste/.a8.conan/.conan_2.0.16/p/b/gtest97ac6ae44faba/b/src/googletest/src/gtest.cc:2672 -15# testing::Test::Run() at /ssh:a8-build-host:/home/gbornste/.a8.conan/.conan_2.0.16/p/b/gtest97ac6ae44faba/b/src/googletest/src/gtest.cc:2677 -16# testing::TestInfo::Run() at /ssh:a8-build-host:/home/gbornste/.a8.conan/.conan_2.0.16/p/b/gtest97ac6ae44faba/b/src/googletest/src/gtest.cc:2836 -17# testing::TestSuite::Run() at /ssh:a8-build-host:/home/gbornste/.a8.conan/.conan_2.0.16/p/b/gtest97ac6ae44faba/b/src/googletest/src/gtest.cc:2968 -18# testing::internal::UnitTestImpl::RunAllTests() at /ssh:a8-build-host:/home/gbornste/.a8.conan/.conan_2.0.16/p/b/gtest97ac6ae44faba/b/src/googletest/src/gtest.cc:5921 -19# bool testing::internal::HandleExceptionsInMethodIfSupported(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) at /ssh:a8-build-host:/home/gbornste/.a8.conan/.conan_2.0.16/p/b/gtest97ac6ae44faba/b/src/googletest/src/gtest.cc:2672 -20# testing::UnitTest::Run() at /ssh:a8-build-host:/home/gbornste/.a8.conan/.conan_2.0.16/p/b/gtest97ac6ae44faba/b/src/googletest/src/gtest.cc:5487 -21# main at /ssh:a8-build-host:/home/gbornste/.a8.conan/.conan_2.0.16/p/b/gtest97ac6ae44faba/b/src/googletest/src/gtest_main.cc:65 -22# __libc_start_call_main at ../sysdeps/nptl/libc_start_call_main.h:58 -23# __libc_start_main at ../csu/libc-start.c:379 -24# _start in ./llfs_Test - -make: *** [Makefile:32: test] Error 6