fix: remove unused attrs field (#41)

* refactor: remove unused attrs field

* chore: cargo fix

* chore: rm enum derive

* fix: validate DeltaOfDelta bytes length in deserialization

Author: Leeeon233

.github/workflows/rust-tests.yml

@@ -0,0 +1,40 @@
+name: Rust Tests
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  test:
+    name: Cargo Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      
+      - name: Install Rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      
+      - name: Cache dependencies
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-cargo-
+      
+      - name: Run tests
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --workspace --all-features 

Cargo.toml

@@ -1,2 +1,14 @@
 [workspace]
 members = ["columnar", "columnar_derive", "fuzz"]
+resolver = "2"
+
+# Added profile settings from fuzz/Cargo.toml
+[profile.release]
+debug = 1
+
+# Added profile settings from columnar/Cargo.toml
+[profile.dev.package.insta]
+opt-level = 3
+
+[profile.dev.package.similar]
+opt-level = 3

README.md

@@ -6,6 +6,8 @@ It allows the contents that need to be serialized and deserialized to be encoded
 
 For more detailed introduction, please refer to this `Notion` link: [Serde-Columnar](https://www.notion.so/loro-dev/Serde-Columnar-Ergonomic-columnar-storage-encoding-crate-7b0c86d6f8d24e4da45a1e2ebd86741c?pvs=4).
 
+![Image](https://github.com/user-attachments/assets/48475c7d-61c8-4903-892d-70d702137dba)
+
 ## 🚧 This crate is in progress and not stable, should not be used in production environments
 
 ## Features 🚀
@@ -30,7 +32,7 @@ Or edit your `Cargo.toml` and add `serde_columnar` as dependency:
 
 ```toml
 [dependencies]
-serde_columnar = "0.3.10"
+serde_columnar = "0.3.11"
 ```
 
 ### Container Attribute

columnar/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "serde_columnar"
-version = "0.3.10"
+version = "0.3.11"
 edition = "2021"
 authors = ["leonzhao <leeeon233@gmail.com>", "zxch3n <remch183@outlook.com>"]
 license = "MIT OR Apache-2.0"
@@ -9,22 +9,23 @@ readme = "README.md"
 repository = "https://github.com/loro-dev/columnar"
 keywords = ["columnar", "column-oriented", "compression", "serde", "compatible"]
 
+[dependencies]
+serde = { version = "1.0" }
+serde_columnar_derive = { path = "../columnar_derive", version = "0.3.6" }
+postcard = { version = "^1.1.0", features = ["alloc"] }
+thiserror = "1.0"
+lazy_static = { version = "1.4", optional = true }
+bincode = { version = "1.3.3", optional = true }
+itertools = "^0.11.0"
+flate2 = { version = "1.0", optional = true }
+
 [dev-dependencies]
 serde = { version = "1.0.188", features = ["derive"] }
 criterion = "0.5.1"
 serde_json = "1.0"
 insta = { version = "1.31.0", features = ["yaml"] }
 rand = "0.8"
 
-[dependencies]
-serde = { version = "1.0" }
-serde_columnar_derive = { path = "../columnar_derive", version = "0.3.5" }
-postcard = { version = "1.0", features = ["alloc"] }
-thiserror = "1.0"
-lazy_static = { version = "1.4", optional = true }
-bincode = { version = "1.3.3", optional = true }
-itertools = "0.11.0"
-flate2 = { version = "1.0", optional = true }
 
 [[bench]]
 name = "columnar_bench"
@@ -43,9 +44,3 @@ required-features = ["analyze"]
 
 [lib]
 doctest = false
-
-[profile.dev.package.insta]
-opt-level = 3
-
-[profile.dev.package.similar]
-opt-level = 3

columnar/src/iterable.rs

@@ -5,6 +5,7 @@ use crate::{
     ColumnarError, DeltaOfDeltaDecoder, DeltaRleable, Rleable,
 };
 use postcard::Deserializer;
+use serde::de::Error;
 use serde::Deserialize;
 
 pub struct GenericIter<'de, T> {
@@ -131,9 +132,8 @@ pub struct DeltaOfDeltaIter<'de, T> {
 
 impl<'de, T: DeltaOfDeltable> DeltaOfDeltaIter<'de, T> {
     pub fn new(bytes: &'de [u8]) -> Self {
-        Self {
-            decoder: DeltaOfDeltaDecoder::new(bytes).unwrap(),
-        }
+        let decoder = DeltaOfDeltaDecoder::new(bytes).unwrap();
+        Self { decoder }
     }
 
     pub(crate) fn try_next(&mut self) -> Result<Option<T>, ColumnarError> {
@@ -301,6 +301,11 @@ impl<'de, T: DeltaOfDeltable> Deserialize<'de> for DeltaOfDeltaIter<'de, T> {
         D: serde::Deserializer<'de>,
     {
         let bytes: &'de [u8] = Deserialize::deserialize(deserializer)?;
+        if bytes.len() < 2 {
+            return Err(D::Error::custom(
+                "DeltaOfDelta bytes should be at least 2 bytes",
+            ));
+        }
         Ok(DeltaOfDeltaIter::new(bytes))
     }
 }

columnar/src/strategy/rle.rs

@@ -528,7 +528,7 @@ impl<'de, T: DeltaOfDeltable> DeltaOfDeltaDecoder<'de, T> {
         let (head_num, bytes) = postcard::take_from_bytes(bytes)?;
         if bytes.is_empty() {
             return Err(ColumnarError::RleDecodeError(
-                "invalid DeltaOfDelta input".to_string(),
+                "Invalid DeltaOfDelta input".to_string(),
             ));
         }
         let last_used_bit = bytes[0];
@@ -543,20 +543,6 @@ impl<'de, T: DeltaOfDeltable> DeltaOfDeltaDecoder<'de, T> {
             last_used_bit,
             _t: PhantomData,
         })
-
-        // let (head_num, bytes) = postcard::take_from_bytes(bytes).unwrap();
-        // let last_used_bit = bytes[0];
-        // let bits = &bytes[1..];
-        // Self {
-        //     bits: &bytes,
-        //     head_num: None,
-        //     prev_value: 0,
-        //     prev_delta: 0,
-        //     index: 0,
-        //     current_bits_index: 0,
-        //     last_used_bit: 0,
-        //     _t: PhantomData,
-        // }
     }
 
     pub fn decode(&mut self) -> Result<Vec<T>, ColumnarError> {
@@ -591,7 +577,11 @@ impl<'de, T: DeltaOfDeltable> DeltaOfDeltaDecoder<'de, T> {
                     self.prev_value += self.prev_delta;
                 }
                 None => return Ok(None),
-                _ => panic!("delta of delta read flag should be 0 or 1"),
+                _ => {
+                    return Err(ColumnarError::RleDecodeError(
+                        "delta of delta read flag should be 0 or 1".to_string(),
+                    ))
+                }
             };
         }
         // println!("prev_value {}", self.prev_value);
@@ -720,12 +710,8 @@ impl<'de, T: DeltaOfDeltable> Iterator for DeltaOfDeltaDecoder<'de, T> {
 
 #[cfg(test)]
 mod test {
-
-    use rand::Rng;
-
-    use crate::column::delta_rle;
-
     use super::{DeltaOfDeltaEncoder, DeltaRleEncoder};
+    use rand::Rng;
 
     #[test]
     fn test_rle() {
@@ -804,8 +790,8 @@ mod test {
             let mut delta_of_delta = DeltaOfDeltaEncoder::new();
             while n < 5000 {
                 if rng.gen_bool(0.93) {
-                    delta_rle.append(i);
-                    delta_of_delta.append(i);
+                    delta_rle.append(i).unwrap();
+                    delta_of_delta.append(i).unwrap();
                     n += 1;
                 }
                 i += 1;

columnar/tests/integration/iterable.rs

@@ -17,7 +17,7 @@ fn row_iterable_strategy() {
         _rle: AnyRleIter::new(&[]),
         _delta_rle: DeltaRleIter::new(&[]),
         _bool_rle: BoolRleIter::new(&[]),
-        _delta_of_delta: DeltaOfDeltaIter::new(&[]),
+        _delta_of_delta: DeltaOfDeltaIter::new(&[0, 0]),
     };
 }
 

columnar/tests/integration/snapshots/tests__integration__attribute__delta_of_delta.snap

@@ -4,9 +4,9 @@ expression: bytes
 ---
 - 1
 - 1
-- 4
+- 5
+- 1
 - 2
 - 2
 - 160
 - 0
-

columnar_derive/Cargo.toml

@@ -1,6 +1,7 @@
 [package]
 name = "serde_columnar_derive"
-version = "0.3.5"
+version = "0.3.6"
+edition = "2021"
 authors = ["leonzhao <leeeon233@gmail.com>", "zxch3n <remch183@outlook.com>"]
 license = "MIT OR Apache-2.0"
 description = "Macros implementation of #[columnar]"
@@ -15,7 +16,7 @@ doctest = false
 proc-macro2 = "1.0"
 quote = "1.0"
 syn = { version = "2.0", features = ["full"] }
-darling = "0.20.10"
+darling = "^0.20.10"
 
 [features]
 analyze = []

columnar_derive/src/args.rs

@@ -29,10 +29,9 @@ pub struct DeriveArgs {
 pub struct FieldArgs {
     /// the name of field
     pub ident: Option<syn::Ident>,
-    pub vis: syn::Visibility,
+    // pub vis: syn::Visibility,
     /// the type of field
     pub ty: Type,
-    pub attrs: Vec<syn::Attribute>,
     // custom attributes
     /// The index of the field in the struct, starts from 0 default.
     pub index: Option<usize>,
@@ -51,21 +50,6 @@ pub struct FieldArgs {
     pub iter: Option<Type>,
 }
 
-#[derive(FromVariant, Debug)]
-#[darling(attributes(columnar))]
-pub struct VariantArgs {
-    // pub ident: syn::Ident,
-    // pub vis: syn::Visibility,
-    // pub ty: syn::Type,
-    pub attrs: Vec<syn::Attribute>,
-    /// the type of the column format, vec or map.
-    #[darling(rename = "class")]
-    pub type_: Option<String>,
-    /// If skip, this field will be ignored.
-    #[darling(default)]
-    pub skip: bool,
-}
-
 #[derive(Debug, Clone, Copy)]
 pub enum Strategy {
     Rle,
@@ -98,11 +82,10 @@ pub enum AsType {
 }
 
 pub trait Args {
-    fn ident(&self) -> Option<syn::Ident>;
+    // fn ident(&self) -> Option<syn::Ident>;
     fn ty(&self) -> Option<syn::Type>;
-    fn attrs(&self) -> &[syn::Attribute];
-    fn index(&self) -> Option<usize>;
-    fn optional(&self) -> bool;
+    // fn index(&self) -> Option<usize>;
+    // fn optional(&self) -> bool;
     fn strategy(&self) -> Strategy;
     fn can_copy(&self) -> bool {
         match self.strategy() {
@@ -147,21 +130,10 @@ pub trait Args {
 }
 
 impl Args for FieldArgs {
-    fn ident(&self) -> Option<syn::Ident> {
-        self.ident.clone()
-    }
     fn ty(&self) -> Option<syn::Type> {
         Some(self.ty.clone())
     }
-    fn attrs(&self) -> &[syn::Attribute] {
-        &self.attrs
-    }
-    fn index(&self) -> Option<usize> {
-        self.index
-    }
-    fn optional(&self) -> bool {
-        self.optional
-    }
+
     fn strategy(&self) -> Strategy {
         Strategy::from_str(self.strategy.clone())
     }
@@ -265,50 +237,6 @@ impl Args for FieldArgs {
     }
 }
 
-impl Args for VariantArgs {
-    fn ident(&self) -> Option<syn::Ident> {
-        None
-    }
-    fn ty(&self) -> Option<syn::Type> {
-        None
-    }
-    fn attrs(&self) -> &[syn::Attribute] {
-        &self.attrs
-    }
-    fn index(&self) -> Option<usize> {
-        None
-    }
-    fn optional(&self) -> bool {
-        false
-    }
-    fn strategy(&self) -> Strategy {
-        Strategy::None
-    }
-    fn class(&self) -> Option<AsType> {
-        match self.type_.as_deref() {
-            Some("vec") => Some(AsType::Vec),
-            Some("map") => Some(AsType::Map),
-            Some(_) => Some(AsType::Other),
-            None => None,
-        }
-    }
-    fn borrow_lifetimes(&self) -> syn::Result<Option<BTreeSet<Lifetime>>> {
-        unimplemented!("Variant have not implemented borrow")
-    }
-
-    fn self_lifetime(&self) -> syn::Result<BTreeSet<Lifetime>> {
-        unimplemented!("Variant have not implemented self_lifetime")
-    }
-
-    fn has_borrow_lifetime(&self) -> bool {
-        false
-    }
-
-    fn lifetime(&self) -> syn::Result<BTreeSet<Lifetime>> {
-        unimplemented!("Variant have not implemented borrow")
-    }
-}
-
 pub fn get_derive_args(args: &[NestedMeta]) -> syn::Result<DeriveArgs> {
     match DeriveArgs::from_list(args) {
         Ok(v) => Ok(v),
@@ -448,8 +376,6 @@ fn collect_lifetimes(ty: &syn::Type, out: &mut BTreeSet<syn::Lifetime>) {
         | syn::Type::ImplTrait(_)
         | syn::Type::Infer(_)
         | syn::Type::Verbatim(_) => {}
-
-        #[cfg_attr(all(test, exhaustive), deny(non_exhaustive_omitted_patterns))]
         _ => {}
     }
 }