feat: add paging for handle with overflow pages

Author: peeeuzin

Cargo.toml

@@ -27,10 +27,7 @@ fs2 = "0.4.3"
 bincode = "1.3.3"
 crc32fast = "1.4.2"
 glob = "0.3.1"
-log = { version = "0.4", features = [
-    "max_level_debug",
-    "release_max_level_warn",
-] }
+log = { version = "0.4", features = ["std"] }
 
 [build-dependencies]
 hooky-rs = "1.0.0"

src/error.rs

@@ -1,35 +1,21 @@
-use std::fmt::Debug;
-
-use glob::PatternError;
+use std::{fmt::Debug, io};
 
 pub enum Error {
-    IoError(std::io::Error),
-    SerializeError(bincode::Error),
-    Deadlock,
-    DatabaseLocked,
-    AlreadyExists(String),
-    NotFound(String),
-    CorruptedData(CorruptedDataError),
+    Io(io::Error),
+    Parsing(bincode::Error),
+    Corrupted(CorruptedDataError),
+    NotEnoughSpace,
     Other(String),
-    Cannot(String),
-    PatternError(PatternError),
 }
 
 impl Debug for Error {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
-            Error::IoError(err) => write!(f, "IO Error: {}", err),
-            Error::Deadlock => write!(f, "Deadlock"),
-            Error::DatabaseLocked => {
-                write!(f, "Database is locked, maybe another instance is running?")
-            }
-            Error::Other(err) => write!(f, "Other Error: {}", err),
-            Error::CorruptedData(err) => write!(f, "{:?}", err),
-            Error::AlreadyExists(message) => write!(f, "{} already exists", message),
-            Error::NotFound(message) => write!(f, "{} not found", message),
-            Error::Cannot(message) => write!(f, "cannot {}", message),
-            Error::SerializeError(error) => write!(f, "serialize error {}", error),
-            Error::PatternError(error) => write!(f, "pattern error {}", error),
+            Error::Io(err) => write!(f, "{err}"),
+            Error::Parsing(err) => write!(f, "{err}"),
+            Error::Corrupted(msg) => write!(f, "Corrupted data: {:?}", msg),
+            Error::NotEnoughSpace => write!(f, "Not enough space"),
+            Error::Other(msg) => write!(f, "{}", msg),
         }
     }
 }

src/lib.rs

@@ -140,7 +140,7 @@ impl DustData {
         let dustdata_config = dustdata_config();
 
         fs::remove_dir_all(dustdata_config.data_path.join(name))
-            .map_err(|_| error::Error::NotFound("collection".to_owned()))?;
+            .map_err(|_| error::Error::Other("collection not found".to_owned()))?;
 
         Ok(())
     }

src/page/io.rs src/page/block.rssrc/page/io.rs renamed to src/page/block.rs

@@ -1,4 +1,3 @@
-use serde::{de::DeserializeOwned, Serialize};
 use std::{
     fs::{self, File, OpenOptions},
     io::{self, Read, Seek, Write},
@@ -10,7 +9,6 @@ use crate::spec::ValueTrait;
 use super::{
     cache::{Cache, CacheBuilder},
     layout::BlockHeader,
-    pager::Pager,
     spec::{PageNumber, BLOCK_HEADER_SIZE, PAGE_SIZE},
     Page,
 };
@@ -30,13 +28,13 @@ use super::{
 /// | Page 04      |
 /// +--------------+
 /// ```
-pub struct BlockIO {
+pub struct Block {
     pub header: BlockHeader,
     file: File,
     cache: Cache,
 }
 
-impl BlockIO {
+impl Block {
     pub fn new<P>(path: P) -> io::Result<Self>
     where
         P: AsRef<Path>,
@@ -56,14 +54,11 @@ impl BlockIO {
         let header = if is_empty(&file)? {
             let header = BlockHeader::new();
 
-            let header_bytes = header.to_vec();
-
-            file.seek(io::SeekFrom::Start(0))?;
-            file.write_all(&header_bytes)?;
+            Self::write_header(&mut file, &header)?;
 
             header
         } else {
-            let header = read_block_header(&mut file)?;
+            let header = Self::read_header(&mut file)?;
 
             if !header.is_valid() {
                 return Err(io::Error::new(
@@ -107,7 +102,7 @@ impl BlockIO {
         Self::from_file(new_file)
     }
 
-    pub fn write_new_page(&mut self, page: &Page) -> io::Result<u64> {
+    pub fn write_new_page(&mut self, page: &Page) -> io::Result<u32> {
         let new_index = self.len()?;
 
         self.write_page(new_index.into(), page)?;
@@ -146,35 +141,27 @@ impl BlockIO {
         Ok(page)
     }
 
-    pub fn page<T>(&mut self, page_index: u64) -> io::Result<Pager<T>>
+    pub fn page<T>(&mut self, page_index: u64) -> io::Result<Page>
     where
         T: ValueTrait,
     {
         let page = self.read_page(page_index)?;
 
-        Ok(Pager::new(self, page, page_index as u32))
-    }
-
-    pub fn write_header(&mut self) -> io::Result<()> {
-        self.file.seek(io::SeekFrom::Start(0))?;
-
-        self.file.write_all(&self.header.to_vec())?;
-
-        Ok(())
+        Ok(page)
     }
 
-    pub fn len(&self) -> io::Result<u64> {
+    pub fn len(&self) -> io::Result<u32> {
         let file_size = self.size()?;
 
-        Ok(file_size - BLOCK_HEADER_SIZE as u64 / PAGE_SIZE as u64)
+        Ok((file_size as u32 - BLOCK_HEADER_SIZE as u32) / PAGE_SIZE as u32)
     }
 
     pub fn is_empty(&self) -> io::Result<bool> {
         Ok(self.len()? == 0)
     }
 
     pub fn exists(&self, page_index: PageNumber) -> io::Result<bool> {
-        Ok(self.len()? > page_index as u64)
+        Ok(self.len()? > page_index)
     }
 
     pub fn index_to_offset(&self, page_index: u64) -> u64 {
@@ -190,6 +177,30 @@ impl BlockIO {
     pub fn sync(&self) -> io::Result<()> {
         self.file.sync_data()
     }
+
+    pub fn persist_header(&mut self) -> io::Result<()> {
+        Self::write_header(&mut self.file, &self.header)?;
+
+        Ok(())
+    }
+
+    fn write_header(file: &mut File, header: &BlockHeader) -> io::Result<()> {
+        file.seek(io::SeekFrom::Start(0))?;
+
+        file.write_all(&header.to_vec())?;
+
+        Ok(())
+    }
+
+    fn read_header(file: &mut File) -> io::Result<BlockHeader> {
+        let mut buffer = [0; BLOCK_HEADER_SIZE];
+
+        file.seek(io::SeekFrom::Start(0))?;
+
+        file.read_exact(&mut buffer)?;
+
+        Ok(BlockHeader::from_slice(&buffer))
+    }
 }
 
 pub fn open_file(path: &Path) -> io::Result<File> {
@@ -208,13 +219,3 @@ pub fn is_empty(file: &File) -> io::Result<bool> {
 
     Ok(metadata.is_file() && metadata.len() == 0)
 }
-
-pub fn read_block_header(file: &mut File) -> io::Result<BlockHeader> {
-    let mut buffer = [0; BLOCK_HEADER_SIZE];
-
-    file.seek(io::SeekFrom::Start(0))?;
-
-    file.read_exact(&mut buffer)?;
-
-    Ok(BlockHeader::from_slice(&buffer))
-}

src/page/cache/mod.rs

@@ -238,19 +238,19 @@ mod cache_tests {
     fn put_test() {
         let mut cache = CacheBuilder::new().max_size(2).build();
 
-        cache.put(0, Page::create(0).unwrap());
-        cache.put(1, Page::create(0).unwrap());
+        cache.put(0, Page::create(0, 0).unwrap());
+        cache.put(1, Page::create(0, 0).unwrap());
 
         assert_eq!(cache.len(), 2);
     }
 
     #[test]
     fn contains_test() {
         let mut cache = CacheBuilder::new().max_size(4).build();
-        let page = Page::create(0).unwrap();
-        let page1 = Page::create(0).unwrap();
-        let page2 = Page::create(0).unwrap();
-        let page3 = Page::create(0).unwrap();
+        let page = Page::create(0, 0).unwrap();
+        let page1 = Page::create(0, 0).unwrap();
+        let page2 = Page::create(0, 0).unwrap();
+        let page3 = Page::create(0, 0).unwrap();
 
         cache.put(0, page.clone());
         cache.put(1, page1.clone());

src/page/layout.rs

@@ -7,7 +7,6 @@ pub struct PageHeader {
     pub lower: LocationOffset,
     pub upper: LocationOffset,
     pub special: LocationOffset,
-    pub next_page: Option<u64>,
     pub flags: u8,
 }
 
@@ -23,6 +22,10 @@ impl PageHeader {
     pub fn has(&self, flag: PageFlag) -> bool {
         self.flags & (flag as u8) != 0
     }
+
+    pub fn not_has(&self, flag: PageFlag) -> bool {
+        self.flags & (flag as u8) == 0
+    }
 }
 
 pub enum PageFlag {
@@ -34,16 +37,16 @@ pub enum PageFlag {
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct BlockHeader {
     pub magic: [u8; 14],
-    pub root_page: Option<PageNumber>,
-    pub first_free_page_overflow: Option<PageNumber>,
+    pub root_page: PageNumber,
+    pub free_page_overflow: PageNumber,
 }
 
 impl Default for BlockHeader {
     fn default() -> Self {
         Self {
             magic: *BLOCK_MAGIC_BYTES,
-            root_page: None,
-            first_free_page_overflow: None,
+            root_page: 0,
+            free_page_overflow: 0,
         }
     }
 }
@@ -66,35 +69,46 @@ impl BlockHeader {
     }
 }
 
-#[derive(Serialize, Deserialize, Debug, Clone, Default)]
-pub struct CellPointerMetadata {
+#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq)]
+pub struct TuplePointerMetadata {
     pub flags: u8,
-    pub has_overflow: bool,
 }
 
-impl CellPointerMetadata {
+impl TuplePointerMetadata {
     pub fn to_vec(&self) -> Vec<u8> {
         bincode::serialize(self).unwrap()
     }
 
     pub fn from_slice(bytes: &[u8]) -> Self {
         bincode::deserialize(bytes).unwrap()
     }
+
+    pub fn set(&mut self, flag: TuplePointerFlags) {
+        self.flags |= flag as u8
+    }
+
+    pub fn unset(&mut self, flag: TuplePointerFlags) {
+        self.flags &= !(flag as u8)
+    }
+
+    pub fn has(&self, flag: TuplePointerFlags) -> bool {
+        self.flags & (flag as u8) != 0
+    }
+
+    pub fn not_has(&self, flag: TuplePointerFlags) -> bool {
+        self.flags & (flag as u8) == 0
+    }
 }
 
-#[derive(Default, PartialEq, Eq, Debug, Clone, Serialize, Deserialize)]
-pub enum CellPointerFlags {
-    #[default]
+#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize)]
+pub enum TuplePointerFlags {
     None = 0b0000_0000,
     Deleted = 0b0000_0001,
+    OverflowItem = 0b0000_0010,
 }
 
-impl From<u8> for CellPointerFlags {
-    fn from(byte: u8) -> Self {
-        match byte {
-            0b0000_0000 => CellPointerFlags::None,
-            0b0000_0001 => CellPointerFlags::Deleted,
-            _ => panic!("Invalid cell pointer metadata: {:#02x}", byte),
-        }
+impl Default for TuplePointerFlags {
+    fn default() -> Self {
+        Self::None
     }
 }