Skip to content

Commit 9acd6d1

Browse files
committed
feat: adding pager
1 parent 74acfec commit 9acd6d1

14 files changed

Lines changed: 1012 additions & 221 deletions

File tree

Cargo.toml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,10 @@ fs2 = "0.4.3"
2727
bincode = "1.3.3"
2828
crc32fast = "1.4.2"
2929
glob = "0.3.1"
30+
log = { version = "0.4", features = [
31+
"max_level_debug",
32+
"release_max_level_warn",
33+
] }
3034

3135
[build-dependencies]
3236
hooky-rs = "1.0.0"

src/btree/mod.rs

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,12 +25,6 @@ use spec::{
2525

2626
pub const MAX_BRANCHING_FACTOR: u16 = 100;
2727

28-
pub trait KeyTrait: Serialize + DeserializeOwned + PartialOrd + Ord + Clone + Debug {}
29-
impl<T: Serialize + DeserializeOwned + PartialOrd + Ord + Clone + Debug> KeyTrait for T {}
30-
31-
pub trait ValueTrait: Serialize + DeserializeOwned + PartialOrd + Clone + Ord + Debug {}
32-
impl<T: Serialize + DeserializeOwned + PartialOrd + Clone + Ord + Debug> ValueTrait for T {}
33-
3428
#[derive(Debug)]
3529
pub struct Search {
3630
/// The page number where the search ended

src/collection/strategies/lock.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ pub struct LockTransaction<'lock, T: ValueTrait> {
1515
pub collection_name: String,
1616
}
1717

18-
impl<'lock, T: ValueTrait> Transaction<T> for LockTransaction<'lock, T> {
18+
impl<T: ValueTrait> Transaction<T> for LockTransaction<'_, T> {
1919
fn rollback(self) {}
2020

2121
fn xid(&self) -> u64 {
@@ -31,7 +31,7 @@ impl<'lock, T: ValueTrait> Transaction<T> for LockTransaction<'lock, T> {
3131
}
3232
}
3333

34-
impl<'a, T: ValueTrait> Drop for LockTransaction<'a, T> {
34+
impl<T: ValueTrait> Drop for LockTransaction<'_, T> {
3535
fn drop(&mut self) {
3636
let dustdata_config = dustdata_config();
3737
let base_path = dustdata_config.data_path.join(&self.collection_name);

src/collection/xact.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ use std::{marker::PhantomData, path::Path, sync::MutexGuard, time};
33
use crate::{
44
btree::{spec::BTreePair, BTree, BTreeIterator, ValueTrait},
55
error::{Error, Result},
6-
page::{io::BlockIO, Page},
6+
page::{io::BlockIO, pager::Pager, Page},
77
};
88

99
use super::{
@@ -25,7 +25,7 @@ impl<T: ValueTrait> TransactionOperations<T> {
2525
}
2626

2727
pub fn write(&mut self, operation: XLogOperation<T>) -> Result<()> {
28-
let mut page: Page<XLogOperation<T>> = self.io.read_page(0).map_err(Error::IoError)?;
28+
let mut page: Pager<XLogOperation<T>> = self.io.page(0).map_err(Error::IoError)?;
2929

3030
page.write(operation)?;
3131

src/lib.rs

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -46,14 +46,15 @@
4646
//! let user = collection.get("user:1").unwrap();
4747
//! ```
4848
49-
pub mod btree;
50-
pub mod collection;
49+
// pub mod btree;
50+
// pub mod collection;
5151
pub mod config;
5252
pub mod error;
5353
pub mod page;
5454
mod serializer;
55+
pub mod spec;
5556

56-
pub use collection::Collection;
57+
// pub use collection::Collection;
5758
pub use config::*;
5859

5960
pub use bincode;
@@ -116,12 +117,13 @@ impl DustData {
116117
///
117118
/// let collection = dustdata.create_collection::<User>("users").unwrap();
118119
/// ```
119-
pub fn collection<T>(&self, name: &str) -> Result<Arc<collection::Collection<T>>>
120-
where
121-
T: Sync + Send + Clone + Debug + Serialize + DeserializeOwned + 'static + Ord,
122-
{
123-
Ok(Arc::new(collection::Collection::new(name)?))
124-
}
120+
/// pub fn collection<T>(&self, name: &str) -> Result<Arc<collection::Collection<T>>>
121+
/// where
122+
/// T: Sync + Send + Clone + Debug + Serialize + DeserializeOwned + 'static + Ord,
123+
/// {
124+
/// Ok(Arc::new(collection::Collection::new(name)?))
125+
/// }
126+
///
125127
126128
/// Drops a collection.
127129
/// ## Arguments

src/page/cache/mod.rs

Lines changed: 265 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,265 @@
1+
use std::collections::HashMap;
2+
3+
use super::{spec::PageNumber, Page};
4+
5+
const PIN_LIMIT_PERCENT: f32 = 50.0;
6+
const MAX_CACHE_SIZE: usize = 1024;
7+
8+
pub enum FrameFlags {
9+
REF = 0b001,
10+
DIRTY = 0b010,
11+
PINNED = 0b100,
12+
}
13+
14+
#[derive(Clone)]
15+
pub struct Frame {
16+
pub page: Page,
17+
pub page_number: PageNumber,
18+
flags: u8,
19+
}
20+
21+
impl Frame {
22+
pub fn new(page: Page, page_number: PageNumber) -> Self {
23+
Self {
24+
page,
25+
page_number,
26+
flags: FrameFlags::REF as u8,
27+
}
28+
}
29+
30+
pub fn set(&mut self, flag: FrameFlags) {
31+
self.flags |= flag as u8
32+
}
33+
34+
pub fn unset(&mut self, flag: FrameFlags) {
35+
self.flags &= !(flag as u8)
36+
}
37+
38+
pub fn has(&self, flag: FrameFlags) -> bool {
39+
self.flags & (flag as u8) != 0
40+
}
41+
}
42+
43+
type FrameNumber = usize;
44+
45+
pub struct CacheBuilder {
46+
max_size: usize,
47+
pin_limit_percent: f32,
48+
}
49+
50+
impl Default for CacheBuilder {
51+
fn default() -> Self {
52+
CacheBuilder::new()
53+
}
54+
}
55+
56+
impl CacheBuilder {
57+
pub fn new() -> Self {
58+
Self {
59+
max_size: MAX_CACHE_SIZE,
60+
pin_limit_percent: PIN_LIMIT_PERCENT,
61+
}
62+
}
63+
64+
pub fn max_size(mut self, size: usize) -> Self {
65+
self.max_size = size;
66+
self
67+
}
68+
69+
pub fn pin_limit_percent(mut self, percent: f32) -> Self {
70+
self.pin_limit_percent = percent;
71+
self
72+
}
73+
74+
pub fn build(self) -> Cache {
75+
Cache {
76+
buffer: Vec::new(),
77+
pages: HashMap::new(),
78+
clock_hand: 0,
79+
pinned: 0,
80+
max_size: self.max_size,
81+
pin_limit_percent: self.pin_limit_percent,
82+
}
83+
}
84+
}
85+
86+
pub struct Cache {
87+
buffer: Vec<Frame>,
88+
pages: HashMap<PageNumber, FrameNumber>,
89+
clock_hand: FrameNumber,
90+
max_size: usize,
91+
92+
pin_limit_percent: f32,
93+
pinned: usize,
94+
}
95+
96+
impl Cache {
97+
pub fn contains(&self, page_number: PageNumber) -> bool {
98+
self.pages.contains_key(&page_number)
99+
}
100+
101+
pub fn len(&self) -> usize {
102+
self.pages.len()
103+
}
104+
105+
pub fn is_empty(&self) -> bool {
106+
self.pages.is_empty()
107+
}
108+
109+
pub fn put(&mut self, page_number: PageNumber, page: Page) -> Option<Frame> {
110+
if let Some(idx) = self.pages.get(&page_number) {
111+
let entry = &mut self.buffer[*idx];
112+
113+
entry.page = page;
114+
115+
return None;
116+
}
117+
118+
if self.buffer.len() < self.max_size {
119+
self.buffer.push(Frame::new(page, page_number));
120+
self.pages.insert(page_number, self.buffer.len() - 1);
121+
122+
None
123+
} else {
124+
self.replace(page_number, page)
125+
}
126+
}
127+
128+
pub fn get(&mut self, page_number: PageNumber) -> Option<&Page> {
129+
match self.pages.get(&page_number) {
130+
None => None,
131+
Some(idx) => {
132+
let frame = &mut self.buffer[*idx];
133+
134+
frame.set(FrameFlags::REF);
135+
Some(&frame.page)
136+
}
137+
}
138+
}
139+
140+
fn get_frame_mut(&mut self, page_number: PageNumber) -> Option<&mut Frame> {
141+
match self.pages.get(&page_number) {
142+
None => None,
143+
Some(idx) => {
144+
let frame = &mut self.buffer[*idx];
145+
146+
Some(frame)
147+
}
148+
}
149+
}
150+
151+
pub fn peek(&mut self, page_number: PageNumber) -> Option<&Page> {
152+
match self.pages.get(&page_number) {
153+
None => None,
154+
Some(idx) => Some(self.buffer.get(*idx).map(|entry| &entry.page).unwrap()),
155+
}
156+
}
157+
158+
fn replace(&mut self, page_number: PageNumber, page: Page) -> Option<Frame> {
159+
let mut attempts = 0;
160+
161+
while attempts < self.pages.len() {
162+
let index = self.clock_hand % self.pages.len();
163+
let frame = &mut self.buffer[index];
164+
165+
if frame.has(FrameFlags::PINNED) {
166+
self.clock_hand = (self.clock_hand + 1) % self.pages.len();
167+
attempts += 1;
168+
continue;
169+
}
170+
171+
if frame.has(FrameFlags::REF) {
172+
frame.unset(FrameFlags::REF);
173+
self.clock_hand = (self.clock_hand + 1) % self.pages.len();
174+
attempts += 1;
175+
176+
continue;
177+
}
178+
179+
let mut dirty_page = None;
180+
if frame.has(FrameFlags::DIRTY) {
181+
dirty_page = Some(frame.clone());
182+
}
183+
184+
*frame = Frame::new(page, page_number);
185+
186+
self.clock_hand = (self.clock_hand + 1) % self.pages.len();
187+
return dirty_page;
188+
}
189+
190+
None
191+
}
192+
193+
pub fn pin(&mut self, page_number: PageNumber) -> bool {
194+
let pinned_percentage = self.pinned as f32 / self.max_size as f32 * 100.0;
195+
196+
if pinned_percentage >= self.pin_limit_percent {
197+
return false;
198+
}
199+
200+
match self.get_frame_mut(page_number) {
201+
None => false,
202+
Some(frame) => {
203+
frame.set(FrameFlags::PINNED);
204+
self.pinned += 1;
205+
true
206+
}
207+
}
208+
}
209+
210+
pub fn unpin(&mut self, page_number: PageNumber) -> bool {
211+
match self.get_frame_mut(page_number) {
212+
None => false,
213+
Some(frame) => {
214+
frame.unset(FrameFlags::PINNED);
215+
self.pinned -= 1;
216+
true
217+
}
218+
}
219+
}
220+
221+
pub fn dirty(&mut self, page_number: PageNumber, page: Page) -> bool {
222+
match self.get_frame_mut(page_number) {
223+
None => false,
224+
Some(frame) => {
225+
frame.page = page;
226+
frame.set(FrameFlags::DIRTY);
227+
true
228+
}
229+
}
230+
}
231+
}
232+
233+
#[cfg(test)]
234+
mod cache_tests {
235+
use super::*;
236+
237+
#[test]
238+
fn put_test() {
239+
let mut cache = CacheBuilder::new().max_size(2).build();
240+
241+
cache.put(0, Page::create(0).unwrap());
242+
cache.put(1, Page::create(0).unwrap());
243+
244+
assert_eq!(cache.len(), 2);
245+
}
246+
247+
#[test]
248+
fn contains_test() {
249+
let mut cache = CacheBuilder::new().max_size(4).build();
250+
let page = Page::create(0).unwrap();
251+
let page1 = Page::create(0).unwrap();
252+
let page2 = Page::create(0).unwrap();
253+
let page3 = Page::create(0).unwrap();
254+
255+
cache.put(0, page.clone());
256+
cache.put(1, page1.clone());
257+
cache.put(2, page2.clone());
258+
cache.put(3, page3.clone());
259+
260+
assert!(cache.contains(0));
261+
assert!(cache.contains(1));
262+
assert!(cache.contains(2));
263+
assert!(cache.contains(3));
264+
}
265+
}

0 commit comments

Comments
 (0)