Skip to content

Commit 8bc4289

Browse files
poyrazKpoyrazK
authored
Merge pull request #16 from poyrazK/feature/benchmarking
perf: add benchmarking infrastructure and performance baseline report
2 parents d023edd + a9d4e6d commit 8bc4289

5 files changed

Lines changed: 342 additions & 0 deletions

File tree

‎CMakeLists.txt‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,16 @@ FetchContent_Declare(
2222
set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
2323
FetchContent_MakeAvailable(googletest)
2424

25+
# Find Google Benchmark
26+
FetchContent_Declare(
27+
googlebenchmark
28+
URL https://github.com/google/benchmark/archive/refs/tags/v1.8.3.zip
29+
)
30+
set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE)
31+
set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "" FORCE)
32+
set(BENCHMARK_ENABLE_GTEST_TESTS OFF CACHE BOOL "" FORCE)
33+
FetchContent_MakeAvailable(googlebenchmark)
34+
2535
# Core Library
2636
set(CORE_SOURCES
2737
src/common/config.cpp
@@ -81,6 +91,12 @@ macro(add_cloudsql_test NAME SOURCE)
8191
add_test(NAME ${NAME} COMMAND ${NAME})
8292
endmacro()
8393

94+
# Benchmark helper macro
95+
macro(add_cloudsql_benchmark NAME SOURCE)
96+
add_executable(${NAME} ${SOURCE})
97+
target_link_libraries(${NAME} sqlEngineCore benchmark::benchmark benchmark::benchmark_main)
98+
endmacro()
99+
84100
# Tests
85101
if(BUILD_TESTS)
86102
enable_testing()
@@ -106,3 +122,11 @@ if(BUILD_TESTS)
106122
COMMAND ${CMAKE_CTEST_COMMAND}
107123
COMMENT "Running all tests via CTest")
108124
endif()
125+
126+
# Benchmarks
127+
option(BUILD_BENCHMARKS "Enable performance benchmarks" OFF)
128+
if(BUILD_BENCHMARKS)
129+
add_cloudsql_benchmark(storage_bench benchmarks/storage_bench.cpp)
130+
add_cloudsql_benchmark(execution_bench benchmarks/execution_bench.cpp)
131+
add_cloudsql_benchmark(network_bench benchmarks/network_bench.cpp)
132+
endif()

‎benchmarks/execution_bench.cpp‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
#include <benchmark/benchmark.h>
2+
#include <memory>
3+
#include <vector>
4+
#include <filesystem>
5+
#include "storage/storage_manager.hpp"
6+
#include "storage/buffer_pool_manager.hpp"
7+
#include "storage/heap_table.hpp"
8+
#include "executor/operator.hpp"
9+
#include "executor/query_executor.hpp"
10+
#include "parser/expression.hpp"
11+
#include "catalog/catalog.hpp"
12+
#include "common/config.hpp"
13+
14+
using namespace cloudsql;
15+
using namespace cloudsql::storage;
16+
using namespace cloudsql::executor;
17+
using namespace cloudsql::parser;
18+
19+
// Helper to create a table with N rows
20+
static void SetupBenchTable(HeapTable& table, int num_rows) {
21+
for (int i = 0; i < num_rows; ++i) {
22+
std::vector<common::Value> values = {
23+
common::Value::make_int64(i),
24+
common::Value::make_text("Data_" + std::to_string(i))
25+
};
26+
table.insert(Tuple(values), 0);
27+
}
28+
}
29+
30+
static void BM_ExecutionSeqScan(benchmark::State& state) {
31+
std::string test_dir = "./bench_exec_scan_" + std::to_string(state.range(0));
32+
std::filesystem::create_directories(test_dir);
33+
StorageManager disk_manager(test_dir);
34+
BufferPoolManager bpm(2000, disk_manager);
35+
36+
Schema schema;
37+
schema.add_column("id", common::ValueType::TYPE_INT64);
38+
schema.add_column("data", common::ValueType::TYPE_TEXT);
39+
40+
for (auto _ : state) {
41+
state.PauseTiming();
42+
auto table = std::make_unique<HeapTable>("scan_table", bpm, schema);
43+
table->create();
44+
SetupBenchTable(*table, state.range(0));
45+
state.ResumeTiming();
46+
47+
auto scan_op = std::make_unique<SeqScanOperator>(std::move(table));
48+
scan_op->init();
49+
Tuple tuple;
50+
while (scan_op->next(tuple)) {
51+
benchmark::DoNotOptimize(tuple);
52+
}
53+
54+
state.PauseTiming();
55+
std::filesystem::remove_all(test_dir);
56+
std::filesystem::create_directories(test_dir);
57+
state.ResumeTiming();
58+
}
59+
60+
state.SetItemsProcessed(state.iterations() * state.range(0));
61+
std::filesystem::remove_all(test_dir);
62+
}
63+
BENCHMARK(BM_ExecutionSeqScan)->Arg(1000)->Arg(10000);
64+
65+
static void BM_ExecutionHashJoin(benchmark::State& state) {
66+
std::string test_dir = "./bench_exec_join_" + std::to_string(state.range(0));
67+
std::filesystem::create_directories(test_dir);
68+
StorageManager disk_manager(test_dir);
69+
BufferPoolManager bpm(4000, disk_manager);
70+
71+
Schema schema;
72+
schema.add_column("id", common::ValueType::TYPE_INT64);
73+
schema.add_column("data", common::ValueType::TYPE_TEXT);
74+
75+
for (auto _ : state) {
76+
state.PauseTiming();
77+
auto left_table = std::make_unique<HeapTable>("left_table", bpm, schema);
78+
left_table->create();
79+
SetupBenchTable(*left_table, state.range(0));
80+
81+
auto right_table = std::make_unique<HeapTable>("right_table", bpm, schema);
82+
right_table->create();
83+
SetupBenchTable(*right_table, state.range(0));
84+
state.ResumeTiming();
85+
86+
auto left_scan = std::make_unique<SeqScanOperator>(std::move(left_table));
87+
auto right_scan = std::make_unique<SeqScanOperator>(std::move(right_table));
88+
89+
// Join on "id"
90+
auto left_key = std::make_unique<ColumnExpr>("id");
91+
auto right_key = std::make_unique<ColumnExpr>("id");
92+
93+
auto join_op = std::make_unique<HashJoinOperator>(
94+
std::move(left_scan), std::move(right_scan), std::move(left_key), std::move(right_key));
95+
96+
join_op->init();
97+
Tuple tuple;
98+
while (join_op->next(tuple)) {
99+
benchmark::DoNotOptimize(tuple);
100+
}
101+
102+
state.PauseTiming();
103+
std::filesystem::remove_all(test_dir);
104+
std::filesystem::create_directories(test_dir);
105+
state.ResumeTiming();
106+
}
107+
108+
state.SetItemsProcessed(state.iterations() * state.range(0));
109+
std::filesystem::remove_all(test_dir);
110+
}
111+
BENCHMARK(BM_ExecutionHashJoin)->Arg(100)->Arg(1000);

‎benchmarks/network_bench.cpp‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
#include <benchmark/benchmark.h>
2+
#include <memory>
3+
#include <vector>
4+
#include <thread>
5+
#include <atomic>
6+
#include <sys/socket.h>
7+
#include <unistd.h>
8+
#include <iostream>
9+
#include "network/rpc_server.hpp"
10+
#include "network/rpc_client.hpp"
11+
#include "network/rpc_message.hpp"
12+
13+
using namespace cloudsql::network;
14+
15+
class NetworkBenchmark : public benchmark::Fixture {
16+
public:
17+
std::unique_ptr<RpcServer> server;
18+
std::unique_ptr<RpcClient> client;
19+
int port = 9000;
20+
21+
void SetUp(const ::benchmark::State& state) override {
22+
port = 9000 + state.range(0); // Different payload sizes on different ports
23+
server = std::make_unique<RpcServer>(port);
24+
25+
server->set_handler(RpcType::AppendEntries, [](const RpcHeader& header, const std::vector<uint8_t>& payload, int client_fd) {
26+
RpcHeader resp_header = header;
27+
resp_header.payload_len = static_cast<uint16_t>(payload.size());
28+
char header_buf[RpcHeader::HEADER_SIZE];
29+
resp_header.encode(header_buf);
30+
31+
if (send(client_fd, header_buf, RpcHeader::HEADER_SIZE, 0) < 0) {
32+
std::cerr << "Handler failed to send header to fd=" << client_fd << " errno=" << errno << std::endl;
33+
return;
34+
}
35+
if (send(client_fd, payload.data(), payload.size(), 0) < 0) {
36+
std::cerr << "Handler failed to send payload to fd=" << client_fd << " errno=" << errno << std::endl;
37+
return;
38+
}
39+
});
40+
41+
if (!server->start()) {
42+
const_cast<::benchmark::State&>(state).SkipWithError("RPC server failed to start");
43+
return;
44+
}
45+
46+
std::this_thread::sleep_for(std::chrono::milliseconds(50));
47+
client = std::make_unique<RpcClient>("127.0.0.1", port);
48+
if (!client->connect()) {
49+
const_cast<::benchmark::State&>(state).SkipWithError("RPC client failed to connect");
50+
return;
51+
}
52+
}
53+
54+
void TearDown(const ::benchmark::State& state) override {
55+
client.reset();
56+
if (server) {
57+
server->stop();
58+
}
59+
server.reset();
60+
}
61+
};
62+
63+
BENCHMARK_DEFINE_F(NetworkBenchmark, RpcRoundTrip)(benchmark::State& state) {
64+
if (!client || !client->is_connected()) {
65+
state.SkipWithError("Client not connected");
66+
return;
67+
}
68+
69+
std::vector<uint8_t> request(state.range(0), 0xAA);
70+
std::vector<uint8_t> response;
71+
72+
for (auto _ : state) {
73+
if (!client->call(RpcType::AppendEntries, request, response)) {
74+
state.SkipWithError("RPC call failed");
75+
break;
76+
}
77+
}
78+
79+
state.SetBytesProcessed(state.iterations() * state.range(0) * 2);
80+
}
81+
BENCHMARK_REGISTER_F(NetworkBenchmark, RpcRoundTrip)->Arg(64)->Arg(1024)->Arg(16384);

‎benchmarks/storage_bench.cpp‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
#include <benchmark/benchmark.h>
2+
#include <memory>
3+
#include <vector>
4+
#include <cstdio>
5+
#include <filesystem>
6+
#include "storage/storage_manager.hpp"
7+
#include "storage/buffer_pool_manager.hpp"
8+
#include "storage/heap_table.hpp"
9+
#include "catalog/catalog.hpp"
10+
#include "common/config.hpp"
11+
12+
using namespace cloudsql;
13+
using namespace cloudsql::storage;
14+
15+
static void BM_BufferPoolPageFetch(benchmark::State& state) {
16+
std::string test_dir = "./bench_data_fetch_" + std::to_string(state.range(0)) + "_" + std::to_string(state.thread_index());
17+
std::filesystem::remove_all(test_dir);
18+
std::filesystem::create_directories(test_dir);
19+
20+
{
21+
StorageManager disk_manager(test_dir);
22+
BufferPoolManager bpm(state.range(0), disk_manager);
23+
24+
std::string file_name = "test_file.db";
25+
disk_manager.open_file(file_name);
26+
27+
// Pre-allocate some pages
28+
std::vector<uint32_t> page_ids;
29+
for (int i = 0; i < 100; ++i) {
30+
uint32_t pid;
31+
bpm.new_page(file_name, &pid);
32+
bpm.unpin_page(file_name, pid, false);
33+
page_ids.push_back(pid);
34+
}
35+
36+
for (auto _ : state) {
37+
for (uint32_t pid : page_ids) {
38+
auto page = bpm.fetch_page(file_name, pid);
39+
if (page) {
40+
bpm.unpin_page(file_name, pid, false);
41+
}
42+
}
43+
}
44+
45+
state.SetItemsProcessed(state.iterations() * page_ids.size());
46+
}
47+
48+
std::filesystem::remove_all(test_dir);
49+
}
50+
BENCHMARK(BM_BufferPoolPageFetch)->Arg(10)->Arg(100)->Arg(1000);
51+
52+
static void BM_HeapTableInsert(benchmark::State& state) {
53+
std::string test_dir = "./bench_data_table_insert_" + std::to_string(state.thread_index());
54+
std::filesystem::remove_all(test_dir);
55+
std::filesystem::create_directories(test_dir);
56+
57+
{
58+
StorageManager disk_manager(test_dir);
59+
BufferPoolManager bpm(1000, disk_manager);
60+
61+
executor::Schema schema;
62+
schema.add_column("id", common::ValueType::TYPE_INT64);
63+
schema.add_column("data", common::ValueType::TYPE_TEXT);
64+
65+
HeapTable table("bench_table", bpm, schema);
66+
table.create();
67+
68+
std::vector<common::Value> values = {
69+
common::Value::make_int64(42),
70+
common::Value::make_text("Benchmark test data string")
71+
};
72+
executor::Tuple tuple(values);
73+
74+
for (auto _ : state) {
75+
table.insert(tuple, 0);
76+
}
77+
78+
state.SetItemsProcessed(state.iterations());
79+
}
80+
81+
std::filesystem::remove_all(test_dir);
82+
}
83+
BENCHMARK(BM_HeapTableInsert);

‎docs/performance/REPORT_V1.md‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# Performance Analysis Report V1.0
2+
3+
## 1. Executive Summary
4+
This report establishes the performance baseline for the `cloudSQL` distributed engine. We established a significant performance delta between Debug and Release builds, and identified a primary bottleneck in the data insertion path related to dynamic memory allocation.
5+
6+
## 2. Baseline Results (Release Build -O3)
7+
8+
| Component | Metric | Baseline Performance |
9+
| :--- | :--- | :--- |
10+
| **Storage** | Buffer Pool Page Fetch | **6.1 Million ops/sec** |
11+
| **Storage** | Heap Table Insertion | **14.0k tuples/sec** |
12+
| **Execution** | SeqScan (10k rows) | **41.8 Billion items/sec** |
13+
| **Execution** | Hash Join (1k rows) | **1.3 Billion items/sec** |
14+
| **Network** | RPC Round-Trip (64B) | **~12μs latency** |
15+
| **Network** | RPC Throughput (16KB) | **2.16 GiB/s throughput** |
16+
17+
## 3. Profiling Findings (`HeapTable::insert`)
18+
19+
Using the macOS `sample` profiler on the `BM_HeapTableInsert` benchmark, we identified the following call graph distribution:
20+
21+
### The "Malloc" Bottleneck
22+
* **65% of CPU time** in the insertion path is spent in `malloc`, `free`, and `operator new`.
23+
* **42% of total insertion time** is attributed to `BufferPoolManager::unpin_page`.
24+
* Within `unpin_page`, the `LRUReplacer::unpin` method is triggering expensive internal allocations within `std::unordered_map` and `std::list`.
25+
26+
### Call Graph Insight:
27+
```plaintext
28+
BM_HeapTableInsert
29+
-> HeapTable::insert (1,859 samples)
30+
-> BufferPoolManager::unpin_page (789 samples)
31+
-> LRUReplacer::unpin (334 samples)
32+
-> std::__hash_table::emplace_unique_key_args
33+
-> operator new
34+
-> malloc_tiny
35+
```
36+
37+
## 4. Conclusion & Recommendations
38+
The system is currently **allocation-bound** for write-heavy workloads. While the execution engine (scans/joins) is highly optimized by the compiler, the storage layer's reliance on standard containers and frequent small allocations during unpinning and tuple creation is limiting throughput.
39+
40+
### Recommended Optimizations:
41+
1. **Tuple Arena/Pool**: Implement a fixed-size memory arena for `Tuple` and `Value` objects to eliminate `malloc` in the insertion hot-path.
42+
2. **Lock-Free / Pre-allocated LRU**: Refactor `LRUReplacer` to use a pre-allocated array-based structure (e.g., a CLOCK algorithm or a fixed-node linked list) to prevent allocations during `unpin`.
43+
3. **Batch Unpinning**: Reduce the frequency of `unpin_page` calls by holding pins for multiple operations where safe.

0 commit comments

Comments
 (0)