Skip to content

Commit f595d9d

Browse files
authored
Merge pull request #4 from 1Cor125/add_global_db_registry
Add global db registry
2 parents 105c736 + fa78d1a commit f595d9d

6 files changed

Lines changed: 287 additions & 28 deletions

File tree

.github/workflows/ci.yml

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,20 @@ jobs:
4141
run: npm ci
4242

4343
- name: Run standards checks
44-
run: npm run standards
45-
env:
46-
# For PRs: lint from base branch. For pushes: lint from previous commit
47-
COMMITLINT_FROM: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
44+
run: |
45+
# Determine commitlint range
46+
if [ "${{ github.event_name }}" = "pull_request" ]; then
47+
export COMMITLINT_FROM="${{ github.event.pull_request.base.sha }}"
48+
else
49+
# For pushes, verify the before commit exists
50+
if git cat-file -e "${{ github.event.before }}" 2>/dev/null; then
51+
export COMMITLINT_FROM="${{ github.event.before }}"
52+
else
53+
# Fallback: check only HEAD commit on force push or first push
54+
export COMMITLINT_FROM="HEAD~1"
55+
fi
56+
fi
57+
npm run standards
4858
4959
- name: Run cargo check
5060
run: cargo check --workspace --all-targets

crates/sqlx-sqlite-conn-mgr/README.md

Lines changed: 51 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,12 @@ management.
1212
Prevents violation of access policies and/or a glut of open file handles and
1313
(mostly) idle threads
1414
* **Connection pooling**:
15-
* Read-only pool for concurrent reads (up to 6 connections)
15+
* Read-only pool for concurrent reads (default: 6 connections, configurable)
1616
* **Lazy write pool**: Single write connection pool (max_connections=1) initialized on
1717
first use
1818
* **Exclusive write access**: WriteGuard ensures serialized writes
19-
* **WAL mode**: Automatically enabled on first `acquire_writer()` call
20-
(idempotent)
19+
* **WAL mode**: Automatically enabled on first `acquire_writer()` call (setting
20+
journal mode to WAL is safe and idempotent)
2121
* See [WAL documentation](https://www.sqlite.org/wal.html) for details
2222
* **30-second idle timeout**: Both read and write connections close after
2323
30 seconds of inactivity
@@ -44,18 +44,19 @@ use std::sync::Arc;
4444
#[tokio::main]
4545
async fn main() -> Result<(), sqlx_sqlite_conn_mgr::Error> {
4646
// Connect to database (creates if missing, returns Arc<SqliteDatabase>)
47-
let db = SqliteDatabase::connect("example.db").await?;
47+
// (See below for how to customize the configuration)
48+
let db = SqliteDatabase::connect("example.db", None).await?;
4849

4950
// Multiple connects to the same path return the same instance
50-
let db2 = SqliteDatabase::connect("example.db").await?;
51+
let db2 = SqliteDatabase::connect("example.db", None).await?;
5152
assert!(Arc::ptr_eq(&db, &db2));
5253

53-
// Use read_pool() for SELECT queries (supports concurrent reads)
54+
// Use read_pool() for read queries (supports concurrent reads)
5455
let rows = query("SELECT * FROM users")
5556
.fetch_all(db.read_pool()?)
5657
.await?;
5758

58-
// Optionally acquire writer for INSERT/UPDATE/DELETE (exclusive access)
59+
// Optionally acquire writer for write queries (exclusive access)
5960
// WAL mode is enabled automatically on first call
6061
let mut writer = db.acquire_writer().await?;
6162
query("INSERT INTO users (name) VALUES (?)")
@@ -70,12 +71,39 @@ async fn main() -> Result<(), sqlx_sqlite_conn_mgr::Error> {
7071
}
7172
```
7273

74+
### Custom Configuration
75+
76+
Only customize the configuration when the defaults don't meet your requirements:
77+
78+
```rust
79+
use sqlx_sqlite_conn_mgr::{SqliteDatabase, SqliteDatabaseConfig};
80+
use std::time::Duration;
81+
82+
#[tokio::main]
83+
async fn main() -> Result<(), sqlx_sqlite_conn_mgr::Error> {
84+
// Only create custom configuration when defaults aren't suitable
85+
let custom_config = SqliteDatabaseConfig {
86+
max_read_connections: 10,
87+
idle_timeout: Duration::from_secs(60),
88+
};
89+
90+
// Pass custom configuration to connect()
91+
let db = SqliteDatabase::connect("example.db", Some(custom_config)).await?;
92+
93+
// Use the database as normal...
94+
db.close().await?;
95+
Ok(())
96+
}
97+
```
98+
7399
## API Overview
74100

75101
### `SqliteDatabase`
76102

77-
* `connect(path)` - Connect to a database (creates if missing, returns cached
78-
`Arc<SqliteDatabase>` if already open)
103+
* `connect(path, custom_config)` - Connect to a database (creates if missing,
104+
returns cached `Arc<SqliteDatabase>` if already open). Pass `None` for
105+
`custom_config` to use defaults (recommended for most use cases), or
106+
`Some(SqliteDatabaseConfig)` when you need to customize the configuration
79107
* `read_pool()` - Get reference to the read-only connection pool for read
80108
operations (returns `Result`)
81109
* `acquire_writer()` - Acquire exclusive write access (returns
@@ -85,6 +113,15 @@ async fn main() -> Result<(), sqlx_sqlite_conn_mgr::Error> {
85113
* `close_and_remove()` - Close and delete all database files (.db, .db-wal,
86114
.db-shm)
87115

116+
### `SqliteDatabaseConfig`
117+
118+
Configuration for connection pool behavior:
119+
120+
* `max_read_connections: u32` - Maximum number of concurrent read connections
121+
(default: 6)
122+
* `idle_timeout: Duration` - How long idle connections remain open before
123+
being closed (default: 30 seconds)
124+
88125
### `WriteGuard`
89126

90127
RAII guard that provides exclusive write access. Automatically returns the
@@ -111,12 +148,11 @@ queries.
111148
is released via `WriteGuard` drop.
112149

113150
5. **Connection Management**:
114-
* Read pool: 6 concurrent connections by default, 0 cached
115-
* Can be configured via `SqliteDatabaseConfig`
116-
* Write pool: max 1 connection, 0 cached
117-
* Idle timeout: 30 seconds for both pools
118-
* Can be configured via `SqliteDatabaseConfig`
119-
* No perpetual caching to minimize idle thread overhead
151+
* Read pool: 6 concurrent connections by default (configurable via `custom_config`)
152+
* Write pool: max 1 connection
153+
* Minimum connections: 0 (no perpetual caching)
154+
* Idle timeout: 30 seconds by default (configurable via `custom_config`)
155+
* Only customize `SqliteDatabaseConfig` when defaults don't meet your needs
120156

121157
## Error Handling
122158

crates/sqlx-sqlite-conn-mgr/src/error.rs

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,3 @@ pub enum Error {
1818
#[error("Database has been closed")]
1919
DatabaseClosed,
2020
}
21-
22-
/// A type alias for Results with our Error type
23-
pub type Result<T> = std::result::Result<T, Error>;

crates/sqlx-sqlite-conn-mgr/src/lib.rs

Lines changed: 51 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,22 +12,68 @@
1212
//!
1313
//! ## Architecture
1414
//!
15-
//! - **Dual pools**: Separate read-only pool (max 6 connections) and write pool (max 1 connection)
15+
//! - **Connection pooling**: Separate read-only pool and write pool with a max of 1 connection
1616
//! - **Lazy WAL mode**: Write-Ahead Logging enabled automatically on first write
1717
//! - **Exclusive writes**: Single-connection write pool enforces serialized write access
1818
//! - **Concurrent reads**: Multiple readers can query simultaneously via the read pool
19-
20-
// TODO: Remove these allows once implementation is complete
21-
#![allow(dead_code)]
19+
//!
20+
//! ## Usage
21+
//!
22+
//! // TODO: Remove this ignore once implementation is complete
23+
//! ```ignore
24+
//! use sqlx_sqlite_conn_mgr::SqliteDatabase;
25+
//! use std::sync::Arc;
26+
//!
27+
//! #[tokio::main]
28+
//! async fn main() -> sqlx_sqlite_conn_mgr::Result<()> {
29+
//! // Connect returns Arc<SqliteDatabase>
30+
//! let db = SqliteDatabase::connect("example.db", None).await?;
31+
//!
32+
//! // Multiple connects to the same path return the same instance
33+
//! let db2 = SqliteDatabase::connect("example.db", None).await?;
34+
//! assert!(Arc::ptr_eq(&db, &db2));
35+
//!
36+
//! // Use read_pool() for read queries (concurrent reads)
37+
//! let rows = sqlx::query("SELECT * FROM users")
38+
//! .fetch_all(db.read_pool()?)
39+
//! .await?;
40+
//!
41+
//! // Optionally acquire writer for write queries (exclusive)
42+
//! // WAL mode is enabled automatically on first call
43+
//! let mut writer = db.acquire_writer().await?;
44+
//! sqlx::query("INSERT INTO users (name) VALUES (?)")
45+
//! .bind("Alice")
46+
//! .execute(&mut *writer)
47+
//! .await?;
48+
//!
49+
//! // Close when done
50+
//! db.close().await?;
51+
//! Ok(())
52+
//! }
53+
//! ```
54+
//!
55+
//! ## Design Principles
56+
//!
57+
//! - Uses sqlx's `SqlitePoolOptions` for all pool configuration
58+
//! - Uses sqlx's `SqliteConnectOptions` for connection flags and configuration
59+
//! - Minimal custom logic - delegates to sqlx wherever possible
60+
//! - Global registry caches new database instances (with their pools) and returns existing ones
61+
//! - WAL mode is enabled lazily only when writes are needed
62+
//!
63+
// TODO: Remove this allow once implementation is complete
2264
#![allow(unused)]
2365

2466
mod config;
2567
mod database;
2668
mod error;
69+
mod registry;
2770
mod write_guard;
2871

2972
// Re-export public types
3073
pub use config::SqliteDatabaseConfig;
3174
pub use database::SqliteDatabase;
32-
pub use error::{Error, Result};
75+
pub use error::Error;
3376
pub use write_guard::WriteGuard;
77+
78+
/// A type alias for Results with our custom Error type
79+
pub type Result<T> = std::result::Result<T, Error>;
Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
//! Global database registry to cache new database instances and return existing ones
2+
3+
use crate::Result;
4+
use crate::database::SqliteDatabase;
5+
use std::collections::HashMap;
6+
use std::future::Future;
7+
use std::path::{Path, PathBuf};
8+
use std::sync::{Arc, OnceLock, Weak};
9+
use tokio::sync::RwLock;
10+
11+
/// Global registry for SQLite databases
12+
static DATABASE_REGISTRY: OnceLock<RwLock<HashMap<PathBuf, Weak<SqliteDatabase>>>> =
13+
OnceLock::new();
14+
15+
fn registry() -> &'static RwLock<HashMap<PathBuf, Weak<SqliteDatabase>>> {
16+
DATABASE_REGISTRY.get_or_init(|| RwLock::new(HashMap::new()))
17+
}
18+
19+
/// Check if a path represents an in-memory SQLite database
20+
///
21+
/// Returns true for `:memory:` and `file::memory:*` URIs
22+
pub fn is_memory_database(path: &Path) -> bool {
23+
let path_str = path.to_str().unwrap_or("");
24+
path_str == ":memory:"
25+
|| path_str.starts_with("file::memory:")
26+
|| path_str.contains("mode=memory")
27+
}
28+
29+
/// Get or open a SQLite database connection
30+
///
31+
/// If a database is already connected, returns the cached instance.
32+
/// Otherwise, calls the provided factory function to create a new connection.
33+
///
34+
/// Special case: `:memory:` databases should not be cached (each is unique)
35+
pub async fn get_or_open_database<F, Fut>(path: &Path, factory: F) -> Result<Arc<SqliteDatabase>>
36+
where
37+
F: FnOnce() -> Fut,
38+
Fut: Future<Output = Result<SqliteDatabase>>,
39+
{
40+
// Skip registry for in-memory databases - always create new
41+
if is_memory_database(path) {
42+
let db = factory().await?;
43+
return Ok(Arc::new(db));
44+
}
45+
46+
// Canonicalize the path for consistent lookups
47+
let canonical_path = canonicalize_path(path)?;
48+
49+
// Try to get existing database with read lock (allows concurrent reads)
50+
{
51+
let registry = registry().read().await;
52+
53+
if let Some(weak) = registry.get(&canonical_path) {
54+
if let Some(db) = weak.upgrade() {
55+
return Ok(db);
56+
}
57+
// Weak reference exists but dead - will be cleaned up in write phase
58+
}
59+
}
60+
61+
// Phase 2: Database not found, acquire write lock
62+
let mut registry = registry().write().await;
63+
64+
// Double-check: another thread might have created it while we waited for write lock
65+
if let Some(weak) = registry.get(&canonical_path) {
66+
if let Some(db) = weak.upgrade() {
67+
return Ok(db);
68+
}
69+
}
70+
71+
// Clean up dead weak references while we have the write lock
72+
registry.retain(|_, weak| weak.strong_count() > 0);
73+
74+
// Now we're sure the database doesn't exist - create it while holding the lock
75+
// This prevents race conditions
76+
let db = factory().await?;
77+
let arc_db = Arc::new(db);
78+
79+
// Cache the new database
80+
registry.insert(canonical_path, Arc::downgrade(&arc_db));
81+
82+
Ok(arc_db)
83+
}
84+
85+
/// Helper to canonicalize a database path
86+
///
87+
/// This function attempts to resolve paths to their canonical form to ensure
88+
/// consistent cache lookups. It handles:
89+
/// - Absolute path resolution
90+
/// - Symlink resolution (when file exists)
91+
/// - Parent directory canonicalization (when file doesn't exist yet)
92+
///
93+
/// Known limitations when file doesn't exist:
94+
/// - Case sensitivity: On case-insensitive filesystems (macOS, Windows), paths
95+
/// differing only in case will be treated as different until the file is created.
96+
/// This could lead to multiple connection pools for the same logical database, at
97+
/// least until the file is created and can be canonicalized properly.
98+
/// - Symlinks in filename: If the filename itself will be a symlink (rare for SQLite),
99+
/// different symlink names won't be resolved until the file exists.
100+
fn canonicalize_path(path: &Path) -> std::io::Result<PathBuf> {
101+
match path.canonicalize() {
102+
Ok(p) => Ok(p),
103+
Err(_) => {
104+
// If path doesn't exist, try to canonicalize parent + filename
105+
let parent = path.parent().unwrap_or_else(|| Path::new("."));
106+
let filename = path
107+
.file_name()
108+
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid path"))?;
109+
let canonical_parent = parent.canonicalize()?;
110+
111+
// Note: We preserve the filename case as provided. On case-insensitive
112+
// filesystems, this means "MyDB.db" and "mydb.db" will create separate
113+
// cache entries until the file exists and can be canonicalized properly.
114+
// This is a known limitation but acceptable since:
115+
// 1. Most apps use consistent casing
116+
// 2. After first connection creates the file, subsequent connects will
117+
// use the canonical (on-disk) case
118+
Ok(canonical_parent.join(filename))
119+
}
120+
}
121+
}
122+
123+
/// Remove a database from the cache
124+
///
125+
/// Special case: `:memory:` databases are never in the registry
126+
///
127+
/// Returns an error if the path cannot be canonicalized
128+
pub async fn uncache_database(path: &Path) -> std::io::Result<()> {
129+
// Skip registry for in-memory databases
130+
if is_memory_database(path) {
131+
return Ok(());
132+
}
133+
134+
// Canonicalize path
135+
let canonical_path = canonicalize_path(path)?;
136+
137+
let mut registry = registry().write().await;
138+
registry.remove(&canonical_path);
139+
Ok(())
140+
}
141+
142+
#[cfg(test)]
143+
mod tests {
144+
use super::*;
145+
146+
#[test]
147+
fn test_canonicalize_path() {
148+
let temp_dir = std::env::temp_dir();
149+
let test_path = temp_dir.join("test.db");
150+
151+
// Test that path is canonicalized to absolute path
152+
let canonical = canonicalize_path(&test_path).unwrap();
153+
assert!(canonical.is_absolute());
154+
155+
// Test relative path
156+
let relative_path = Path::new("./test_relative.db");
157+
let canonical_relative = canonicalize_path(relative_path).unwrap();
158+
assert!(canonical_relative.is_absolute());
159+
}
160+
161+
#[test]
162+
fn test_canonicalize_nonexistent_path() {
163+
let temp_dir = std::env::temp_dir();
164+
let nonexistent = temp_dir.join("nonexistent_dir").join("test.db");
165+
166+
// Should fail if parent directory doesn't exist
167+
let result = canonicalize_path(&nonexistent);
168+
assert!(result.is_err());
169+
}
170+
}

0 commit comments

Comments
 (0)