add sequencer

Author: antheas

src/pasteur/metadata.py

@@ -137,6 +137,7 @@ class TableMeta:
 
     def __init__(self, meta: dict):
         self.primary_key = meta.get("primary_key", None)
+        self.sequencer: tuple[str] | str | None = meta.get("sequencer", None)
 
         if "metrics" in meta:
             metrics_dict = meta["metrics"]

src/pasteur/table.py

@@ -268,6 +268,32 @@ def fit_chunk(
             ), "Properly formatted datasets should have their primary key as their index column"
             # table.reindex(meta.primary_key)
 
+        # Process sequencer first
+        seq_name = meta.sequencer
+        if seq_name:
+            col = meta.cols[seq_name]
+            assert (
+                col.type in self.transformer_cls
+            ), f"Column type {col.type} not in transformers:\n{list(self.transformer_cls.keys())}"
+
+            # Fit transformer
+            if "main_param" in col.args:
+                t = self.transformer_cls[col.type].build(
+                    col.args["main_param"], **col.args
+                )
+            else:
+                t = self.transformer_cls[col.type].build(**col.args)
+
+            assert isinstance(t, SeqTransformer), f"Sequencer must be of type 'SeqTransformer', not '{type(t)}'"
+            
+            # Add foreign column if required
+            ref_cols = _calc_unjoined_refs(self.name, get_table, col.ref)
+            res = t.fit(table[seq_name], ref_cols, loaded_ids)
+            assert res
+            seq_attr, seq = res
+        else:
+            seq_attr = seq = None
+
         for name, col in meta.cols.items():
             if col.is_id():
                 continue
@@ -287,7 +313,7 @@ def fit_chunk(
             if isinstance(t, SeqTransformer):
                 # Add foreign column if required
                 ref_cols = _calc_unjoined_refs(self.name, get_table, col.ref)
-                t.fit(table[name], ref_cols, loaded_ids)
+                t.fit(table[name], ref_cols, loaded_ids, seq_attr, seq)
             elif isinstance(t, RefTransformer):
                 # Add foreign column if required
                 ref_cols = _calc_joined_refs(self.name, get_table, loaded_ids, col.ref)
@@ -327,7 +353,29 @@ def transform_chunk(
         tts = []
         ctxs = defaultdict(list)
 
+        # Process sequencer first
+        seq_name = meta.sequencer
+        if seq_name:
+            col = meta.cols[seq_name]
+            trn = self.transformers[seq_name]
+            assert isinstance(trn, SeqTransformer)
+            ref_cols = _calc_unjoined_refs(self.name, get_table, col.ref)
+            assert loaded_ids is not None
+
+            res = trn.transform(table[seq_name], ref_cols, loaded_ids)
+            assert len(res) == 3
+            tt, ctx, seq = res
+
+            for n, c in ctx.items():
+                ctxs[n].append(c)
+        else:
+            seq = None
+
         for name, col in meta.cols.items():
+            # Skip sequencer
+            if seq_name == name:
+                continue
+
             if col.is_id():
                 continue
 
@@ -336,7 +384,9 @@ def transform_chunk(
                 # Add foreign column if required
                 ref_cols = _calc_unjoined_refs(self.name, get_table, col.ref)
                 assert loaded_ids is not None
-                tt, ctx = trn.transform(table[name], ref_cols, loaded_ids)
+                res = trn.transform(table[name], ref_cols, loaded_ids, seq)
+                tt = res[0]
+                ctx = res[1]
 
                 for n, c in ctx.items():
                     ctxs[n].append(c)

src/pasteur/transform.py

@@ -101,14 +101,22 @@ class SeqTransformer(Transformer):
     Sequence Transformers receive unprocessed parent columns, references and the ID table.
     Then, it is up to them to process the data and return the encoded version.
     They can also push columns upstream to parents, through context tables.
+
+    Event-based data is sequential. The Sequential transformers may require the
+    order of each row. For this case, the main Sequence Transformer, which is named 
+    the sequencer, is processed first and returns an additional data column and 
+    attribute during fitting. This column and attribute are fed to the other
+    sequence transformers. 
     """
 
     def fit(
         self,
         data: pd.Series | pd.DataFrame,
         ref: dict[str, pd.DataFrame] | None = None,
         ids: pd.DataFrame | None = None,
-    ) -> tuple[Attributes, dict[str, Attributes]] | None:
+        seq_attr: Attribute | None = None,
+        seq: pd.Series | None = None,
+    ) -> tuple[Attribute, pd.Series] | None:
         pass
 
     def reduce(self, other: "SeqTransformer"):
@@ -122,16 +130,19 @@ def fit_transform(
         data: pd.Series | pd.DataFrame,
         ref: dict[str, pd.DataFrame] | None = None,
         ids: pd.DataFrame | None = None,
-    ) -> tuple[pd.DataFrame, dict[str, pd.DataFrame]]:
-        self.fit(data, ref)
-        return self.transform(data, ref)
+        seq_attr: Attribute | None = None,
+        seq: pd.Series | None = None,
+    ) -> tuple[pd.DataFrame, dict[str, pd.DataFrame]] | tuple[pd.DataFrame, dict[str, pd.DataFrame], pd.Series]:
+        self.fit(data, ref, ids, seq_attr, seq)
+        return self.transform(data, ref, ids, seq)
 
     def transform(
         self,
         data: pd.Series | pd.DataFrame,
         ref: dict[str, pd.DataFrame] | None = None,
         ids: pd.DataFrame | None = None,
-    ) -> tuple[pd.DataFrame, dict[str, pd.DataFrame]]:
+        seq: pd.Series | None = None,
+    ) -> tuple[pd.DataFrame, dict[str, pd.DataFrame]] | tuple[pd.DataFrame, dict[str, pd.DataFrame], pd.Series]:
         raise NotImplementedError()
 
     def reverse(
@@ -140,6 +151,7 @@ def reverse(
         ctx: dict[str, pd.DataFrame],
         ref: dict[str, pd.DataFrame] | None = None,
         ids: pd.DataFrame | None = None,
+        seq: pd.Series | None = None,
     ) -> pd.DataFrame:
         """When reversing, the data column contains encoded data, whereas the ref
         column contains decoded/original data. Therefore, the referred columns have