Ditch DataFrames.jl dependency, implements Tables.jl interface instead

Author: VEZY

Project.toml

@@ -5,7 +5,6 @@ version = "0.14.4"
 
 [deps]
 AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
-DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
 Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
@@ -19,7 +18,6 @@ XLSX = "fdbf4ff8-1666-58a4-91e7-1b58723a45e0"
 
 [compat]
 AbstractTrees = "0.4"
-DataFrames = "1"
 DelimitedFiles = "1"
 Graphs = "1"
 MetaGraphsNext = "0.5, 0.6, 0.7, 0.8"

README.md

@@ -53,12 +53,15 @@ write_mtg("test.mtg",mtg)
 
 ## 3. Conversion
 
-You can convert an MTG into a DataFrame and select the variables you need:
+You can expose an MTG as a `Tables.jl` source:
 
 ```julia
-DataFrame(mtg, [:length_mm, :XX])
+mtg_table(mtg)
+symbol_table(mtg, :Leaf)
 ```
 
+If you use `DataFrames.jl`, `DataFrame(mtg)` works out of the box through the `Tables.jl` interface.
+
 Or convert it to a [MetaGraph](https://juliagraphs.org/MetaGraphsNext.jl/dev/):
 
 ```julia

docs/src/api.md

@@ -7,7 +7,7 @@ Notable attribute APIs:
 - `attribute`, `attribute!`, `attributes`, `attribute_names`
 - `add_column!`, `drop_column!`, `rename_column!`
 - `descendants_strategy`, `descendants_strategy!` (choose automatic/direct/indexed descendants retrieval)
-- `symbol_table`, `mtg_table` (`Tables.jl` sources)
+- `to_table`, `symbol_table`, `mtg_table` (`Tables.jl` sources)
 
 ```@index
 ```

docs/src/get_started.md

@@ -42,18 +42,24 @@ Then you can compute new variables in the MTG using [`transform!`](@ref):
 transform!(mtg, :Length => (x -> isnothing(x) ? nothing : x * 1000.) => :length_mm)
 ```
 
-The design of [`transform!`](@ref) is heavily inspired from the eponym function from [`DataFrame.jl`](https://dataframes.juliadata.org/stable/), with little tweaks for MTGs.
+The design of [`transform!`](@ref) is heavily inspired from the eponym function from tabular workflows (notably [`DataFrames.jl`](https://dataframes.juliadata.org/stable/)), with little tweaks for MTGs.
 
 You can see the newly-computed attributes using descendants like so:
 
 ```@example usepkg
 descendants(mtg, :length_mm)
 ```
 
-Or by transforming your MTG into a DataFrame:
+Or by getting a tabular view of your MTG:
 
 ```@example usepkg
-DataFrame(mtg, :length_mm)
+mtg_table(mtg)
+```
+
+Or directly transforming the MTG into a DataFrame:
+
+```@example usepkg
+DataFrame(mtg)
 ```
 
 Then you can write the MTG back to disk like so:

docs/src/tutorials/2.descendants_ancestors_filters.md

@@ -84,10 +84,10 @@ If you start from another node you can retrieve the root node using [`get_root`]
 get_attributes(get_root(node_5))
 ```
 
-A more simple way to get all nodes and their attributes is to convert the MTG into a DataFrame like so:
+A simple way to get all nodes and their attributes is to use the unified table view:
 
 ```@example usepkg
-DataFrame(mtg)
+mtg_table(mtg)
 ```
 
 ## Descendants

docs/src/tutorials/3.transform_mtg.md

@@ -108,10 +108,10 @@ We can also get its values by using [`descendants`](@ref) on the root node:
 descendants(mtg, :length_m)
 ```
 
-We can also get the values in the form of a DataFrame instead:
+We can also inspect values through the MTG table view:
 
 ```@example usepkg
-DataFrame(mtg, :length_m)
+mtg_table(mtg)
 ```
 
 We can also provide several input variables if we need:
@@ -125,7 +125,7 @@ Here we provide the input attributes as a Vector of Symbols (could be String als
 Let's see the results:
 
 ```@example usepkg
-DataFrame(mtg, [:Length, :width, :volume_cm3])
+mtg_table(mtg)
 ```
 
 The new name of the attribute (the RHS) is optional though. We could write our first example as:
@@ -241,15 +241,15 @@ mtg_select = deepcopy(mtg)
 
 select!(mtg_select, :Length => (x -> x / 10) => :length_m, :Width, ignore_nothing = true)
 
-DataFrame(mtg_select)
+mtg_table(mtg_select)
 ```
 
 There is also a copy-returning version of the function (`select`, without `!`):
 
 ```@example usepkg
 mtg_select = select(mtg, :Length => (x -> x / 10) => :length_m, :Width, ignore_nothing = true)
 
-DataFrame(mtg_select)
+mtg_table(mtg_select)
 ```
 
 ## Traverse an MTG
@@ -298,7 +298,7 @@ This package provides an implementation of the pipe model, used as follows:
 first_cross_section = 0.34 # the initial cross-section of the plant
 
 transform!(mtg, (node -> pipe_model!(node, first_cross_section)) => :cross_section_pipe)
-DataFrame(mtg, :cross_section_pipe)
+descendants(mtg, :cross_section_pipe)
 ```
 
 For more information about the implementation, you can check the documentation of the function: [`pipe_model!`](@ref).

docs/src/tutorials/4.convert_mtg.md

@@ -8,30 +8,36 @@ mtg = read_mtg(file)
 
 We can do a lot using the MTG format, but sometimes we want our data in another format.
 
-That's why `MultiScaleTreeGraph.jl` provide functions to convert an MTG into a DataFrame or into a Graph.
+That's why `MultiScaleTreeGraph.jl` provides functions to expose an MTG as a `Tables.jl` source, or convert it into a graph.
 
-## MTG to DataFrame
+## MTG to table
 
-To convert an MTG into a DataFrame, you can simply use this command:
+To get a unified table view of the MTG, you can use:
 
 ```@example usepkg
-df = DataFrame(mtg, :Width)
+tbl = mtg_table(mtg)
 ```
 
-This will convert your MTG into a DataFrame along with the selected variable (here the Width). The node MTG is always reported in new columns:
+The unified table reports MTG topology columns plus all attributes:
 
-- tree: a pretty-printing of the MTG
-- id: the unique ID of the node in the whole MTG
+- node_id: the unique ID of the node in the whole MTG
 - symbol: the node symbol
 - scale: the node scale
 - index: the node index
 - parent_id: the node's parent id
 - link: the link between the node and its parent
 
-It is also possible to get several attributes as columns by passing their names as a vector:
+It is also possible to get a per-symbol table:
 
 ```@example usepkg
-DataFrame(mtg, [:Width, :Length])
+leaf_tbl = symbol_table(mtg, :Leaf)
+```
+
+If you use `DataFrames.jl` in your own project, `DataFrame(mtg)` still works because MTGs implement the `Tables.jl` interface.
+
+```julia
+using DataFrames
+df = DataFrame(mtg)
 ```
 
 ## MTG to MetaGraph

docs/src/tutorials/6.add_remove_nodes.md

@@ -70,7 +70,7 @@ mtg
 And the attributes:
 
 ```@example usepkg
-DataFrame(mtg, get_attributes(mtg))
+mtg_table(mtg)
 ```
 
 ### Inserting nodes
@@ -332,7 +332,7 @@ insert_siblings!(
 Let's see the results for the area of our leaves:
 
 ```@example usepkg
-DataFrame(mtg_5, :area)
+descendants(mtg_5, :area, self = true)
 ```
 
 ### Write the MTG

src/MultiScaleTreeGraph.jl

@@ -8,17 +8,14 @@ using OrderedCollections
 import XLSX: readxlsx, sheetnames
 import SHA: sha1 # for naming the cache variable
 import Base
-import DataFrames: DataFrame, insertcols!
-import DataFrames: transform!, transform # We define our own version for transforming the MTG
-import DataFrames: select!, select # We define our own version for transforming the MTG
-import DataFrames: rename! # We define our own version for renaming node attributes
 import Tables
 import MetaGraphsNext: MetaGraph, code_for, add_edge! # Transform to MetaGraph
 import Graphs.DiGraph
 import Dates: Date, @dateformat_str, format
 
 include("types/AbstractNodeMTG.jl")
 include("types/Attributes.jl")
+include("types/ColumnTable.jl")
 include("types/Node.jl")
 include("read_MTG/read_MTG.jl")
 include("read_MTG/strip_comments.jl")
@@ -50,7 +47,6 @@ include("compute_MTG/mutation_helpers.jl")
 include("compute_MTG/nleaves.jl")
 include("compute_MTG/pipe_model.jl")
 include("compute_MTG/get_node.jl")
-include("conversion/DataFrame.jl")
 include("conversion/Tables.jl")
 include("conversion/MetaGraph.jl")
 
@@ -68,7 +64,6 @@ export nextsibling, prevsibling, lastsibling
 export print
 export show
 export length
-export DataFrame
 export MetaGraph
 export iterate
 export siblings
@@ -102,6 +97,7 @@ export add_column!, drop_column!, rename_column!
 export descendants_strategy, descendants_strategy!
 export columnarize!
 export symbol_table, mtg_table
+export to_table
 export list_nodes
 export get_classes
 export get_description

src/compute_MTG/summary.jl

@@ -6,13 +6,24 @@ Compute the mtg classes based on its content. Usefull after having mutating the
 function get_classes(mtg)
     attributes = traverse(mtg, node -> (SYMBOL=symbol(node), SCALE=scale(node)), type=@NamedTuple{SYMBOL::Symbol, SCALE::Int64})
     attributes = unique(attributes)
-    df = DataFrame(attributes)
+    n = length(attributes)
+    symbols_ = Vector{Symbol}(undef, n)
+    scales_ = Vector{Int}(undef, n)
+    @inbounds for i in eachindex(attributes)
+        symbols_[i] = attributes[i].SYMBOL
+        scales_[i] = attributes[i].SCALE
+    end
 
-    # Make everything to default values:
-    df[!, :DECOMPOSITION] .= "FREE"
-    df[!, :INDEXATION] .= "FREE"
-    df[!, :DEFINITION] .= "IMPLICIT"
-    return df
+    ColumnTable(
+        Symbol[:SYMBOL, :SCALE, :DECOMPOSITION, :INDEXATION, :DEFINITION],
+        AbstractVector[
+            symbols_,
+            scales_,
+            fill("FREE", n),
+            fill("FREE", n),
+            fill("IMPLICIT", n)
+        ]
+    )
 end
 
 """
@@ -31,53 +42,40 @@ Compute the mtg features section based on its attributes. Usefull after having c
 in the mtg.
 """
 function get_features(mtg)
-    attributes = traverse(
-        mtg,
-        node -> (collect(keys(node_attributes(node))), [typeof(i) for i in values(node_attributes(node))]), type=Tuple{Vector{Symbol},Vector{DataType}}
-    ) |> unique
-
-    df = DataFrame(
-        :NAME => vcat([i[1] for i in attributes]...),
-        :TYPE => vcat([i[2] for i in attributes]...)
-    )
-
-    # filter-out the attributes that have more than one value inside them (not compatible with
-    # the mtg format yet):
-    filter!(
-        x -> !(x.TYPE <: Vector) &&
-                 !(x.TYPE <: Nothing) &&
-                 !in(x.NAME, [:description, :symbols, :scales]),
-        df
-    )
+    names_ = Symbol[]
+    types_ = String[]
+    seen = Set{Tuple{Symbol,String}}()
 
-    # Remove repeated rows:
-    unique!(df)
-
-    new_type = fill("", size(df)[1])
-    for (index, value) in enumerate(df.TYPE)
-        if value <: AbstractFloat
-            new_type[index] = "REAL"
-        elseif value <: Bool
-            # we test booleans before integers because Bool <: Integer
-            new_type[index] = "BOOLEAN"
-        elseif value <: Integer
-            new_type[index] = "INT"
-            # elseif df.NAME[i] in () # Put reserved keywords here if needed in the future
-            # new_type[index] = "ALPHA"
-        elseif value <: Date
-            new_type[index] = "DD/MM/YY"
-        else
-            # All others are parsed as string
-            new_type[index] = "STRING"
+    traverse!(mtg) do node
+        for (name, value) in pairs(node_attributes(node))
+            T = typeof(value)
+            if (T <: AbstractVector) || (T <: Nothing) || (name in (:description, :symbols, :scales))
+                continue
+            end
+
+            typ =
+                if T <: AbstractFloat
+                    "REAL"
+                elseif T <: Bool
+                    "BOOLEAN"
+                elseif T <: Integer
+                    "INT"
+                elseif T <: Date
+                    "DD/MM/YY"
+                else
+                    "STRING"
+                end
+
+            row = (Symbol(name), typ)
+            if !(row in seen)
+                push!(seen, row)
+                push!(names_, row[1])
+                push!(types_, row[2])
+            end
         end
     end
 
-    df.TYPE = new_type
-
-    # Remove repeated rows again after having changed the type (can have String and SubString for the same variable before): 
-    unique!(df)
-
-    return df
+    ColumnTable(Symbol[:NAME, :TYPE], AbstractVector[names_, types_])
 end
 
 """