From 9ef38becae7bcd0649c48f6c0c2f57151fe158f2 Mon Sep 17 00:00:00 2001 From: Vladimir Sibirov Date: Sun, 4 May 2025 20:28:36 +0200 Subject: [PATCH 01/18] Change node instance configuration to Flyde 1.0 format --- examples/HelloPy.flyde | 42 +++----- flyde/flow.py | 68 ++++++++---- flyde/flow.pyi | 11 +- flyde/io.py | 56 ++++++++-- flyde/io.pyi | 18 ++++ flyde/node.py | 163 ++++++++++++++++++++--------- flyde/node.pyi | 52 +++++++-- pyproject.toml | 1 + ruff.toml | 1 + tests/{__init.py__ => __init__.py} | 0 tests/test_io.py | 60 ++++++++++- tests/test_node.py | 106 +++++++++++++++---- 12 files changed, 439 insertions(+), 139 deletions(-) create mode 100644 ruff.toml rename tests/{__init.py__ => __init__.py} (100%) diff --git a/examples/HelloPy.flyde b/examples/HelloPy.flyde index 350eae9..d0fa0a7 100644 --- a/examples/HelloPy.flyde +++ b/examples/HelloPy.flyde @@ -1,34 +1,26 @@ -imports: - "@flyde/stdlib": - - InlineValue - mylib/components.flyde.ts: - - Print +imports: {} node: instances: - pos: - x: -157.29365234375 - y: -130.58668701171877 + x: -156.29365234375 + y: -256.58668701171877 id: Print-g7039qo - inputConfig: {} + inputConfig: + __trigger: + mode: queue + msg: + mode: sticky + visibleInputs: [] nodeId: Print - - pos: - x: -198.96331298828125 - y: -295.8833435058594 - id: vzy5s9fyaacybwutaijgttv1 - inputConfig: {} - nodeId: InlineValue__vzy5s9fyaacybwutaijgttv1 - macroId: InlineValue - macroData: - value: + config: + msg: type: string - value: Hello Flyde! - connections: - - from: - insId: vzy5s9fyaacybwutaijgttv1 - pinId: value - to: - insId: Print-g7039qo - pinId: msg + value: Hello, Flyde! + type: code + source: + type: file + data: mylib/components.flyde.ts + connections: [] id: Example inputs: {} outputs: {} diff --git a/flyde/flow.py b/flyde/flow.py index 1f04796..a0cde98 100644 --- a/flyde/flow.py +++ b/flyde/flow.py @@ -2,20 +2,23 @@ import logging import os import sys +from threading import Event from typing import Callable + import yaml # type: ignore -from threading import Event -from flyde.node import Graph +from flyde.node import Graph, InstanceArgs, InstanceType logger = logging.getLogger(__name__) class Flow: """Flow is a root-level runnable directed acyclic graph of nodes.""" + def __init__(self, imports: dict[str, list[str]]): self._imports = imports self._path = "" + self._base_path = "" self._node: Graph self._components: dict[str, Callable] = {} self._graphs: dict[str, dict] = {} @@ -36,31 +39,59 @@ def _preload_imports(self, base_path: str, imports: dict[str, list[str]]): self._graphs[node_id] = yml["node"] continue # Translate typescript file path to python module - module = ( - module.replace("/", ".").replace(".flyde.ts", "").replace("@", "") - ) + module = module.replace("/", ".").replace(".flyde.ts", "").replace("@", "") logger.debug(f"Importing module {module}") mod = importlib.import_module(module) for class_name in classes: logger.debug(f"Importing {class_name} from {module}") self._components[class_name] = getattr(mod, class_name) - def factory(self, class_name: str, args: dict): + def _load_graph(self, name: str, path: str): + """Loads a graph YAML.""" + yml = load_yaml_file(path) + if not isinstance(yml, dict): + raise ValueError(f"Invalid YAML file {path}") + # Save the blueprint YAML for the graph to be instantiated later + self._graphs[name] = yml["node"] + return + + def _load_component(self, name: str, path: str): + """Loads a component from a Python module.""" + # Translate typescript file path to python module + path = path.replace("/", ".").replace(".flyde.ts", "").replace("@", "") + logger.debug(f"Importing module {path}") + mod = importlib.import_module(path) + self._components[name] = getattr(mod, name) + + def create_graph(self, name: str, args: InstanceArgs): + if name not in self._graphs: + if args.source is None: + raise ValueError(f"Graph {name} does not have a valid source") + self._load_graph(name, args.source.data) + + # Merge the blueprint YAML with the arguments + yml = self._graphs[name] | args.to_dict() + node = Graph.from_yaml(self.factory, yml) + return node + + def create_component(self, name: str, args: InstanceArgs): + if name not in self._components: + if args.source is None: + raise ValueError(f"Component {name} does not have a valid source") + self._load_component(name, args.source.data) + + # Create the component instance + component = self._components[name] + return component(**args.to_dict()) + + def factory(self, class_name: str, args: InstanceArgs): """Factory method to create a node from a class name and arguments. It is used by the runtime to create nodes from the YAML definition or on the fly.""" - if class_name in self._graphs: - # Merge the blueprint YAML with the arguments - yml = self._graphs[class_name] | args - node = Graph.from_yaml(self.factory, yml) - return node - - # Look up the class in the imports - if class_name in self._components: - component = self._components[class_name] - return component(**args) + if args.type == InstanceType.VISUAL: + return self.create_graph(class_name, args) - raise ValueError(f"Unknown class name: {class_name}") + return self.create_component(class_name, args) def run(self): """Start the flow running. This is a non-blocking call as the flow runs in a separate thread.""" @@ -91,7 +122,8 @@ def from_yaml(cls, path: str, yml: dict): raise ValueError("No node in flow definition") ins = cls(imports) - ins._preload_imports(os.path.dirname(path), yml.get("imports", {})) + ins._path = path + ins._base_path = os.path.dirname(path) ins._node = Graph.from_yaml(ins.factory, yml["node"]) ins._node.stopped = Event() return ins diff --git a/flyde/flow.pyi b/flyde/flow.pyi index 68d78c3..27ac0fd 100644 --- a/flyde/flow.pyi +++ b/flyde/flow.pyi @@ -1,5 +1,5 @@ from _typeshed import Incomplete -from flyde.node import Graph as Graph +from flyde.node import Graph as Graph, InstanceArgs as InstanceArgs, InstanceType as InstanceType from threading import Event logger: Incomplete @@ -8,12 +8,19 @@ class Flow: """Flow is a root-level runnable directed acyclic graph of nodes.""" _imports: Incomplete _path: str + _base_path: str _node: Incomplete _components: Incomplete _graphs: Incomplete def __init__(self, imports: dict[str, list[str]]) -> None: ... def _preload_imports(self, base_path: str, imports: dict[str, list[str]]): ... - def factory(self, class_name: str, args: dict): + def _load_graph(self, name: str, path: str): + """Loads a graph YAML.""" + def _load_component(self, name: str, path: str): + """Loads a component from a Python module.""" + def create_graph(self, name: str, args: InstanceArgs): ... + def create_component(self, name: str, args: InstanceArgs): ... + def factory(self, class_name: str, args: InstanceArgs): """Factory method to create a node from a class name and arguments. It is used by the runtime to create nodes from the YAML definition or on the fly.""" diff --git a/flyde/io.py b/flyde/io.py index ca5ef9e..d2e9a8e 100644 --- a/flyde/io.py +++ b/flyde/io.py @@ -1,7 +1,8 @@ from copy import deepcopy +from dataclasses import dataclass from enum import Enum -from typing import Any, Optional from queue import Queue +from typing import Any, Optional EOF = Exception("__EOF__") """EOF is a signal to indicate the end of data.""" @@ -12,6 +13,24 @@ def is_EOF(value: Any) -> bool: return isinstance(value, Exception) and value.args[0] == "__EOF__" +class InputType(Enum): + """Input type contains all input types supported by Flyde.""" + + DYNAMIC = "dynamic" + NUMBER = "number" + BOOLEAN = "boolean" + JSON = "json" + STRING = "string" + + +@dataclass +class InputConfig: + """Configuration of an input in a Flyde flow.""" + + type: InputType + value: Any + + class InputMode(Enum): """InputMode is the mode of an input. @@ -115,14 +134,14 @@ def value(self, value: Any): def get(self) -> Any: """Get the value of the input from either the queue or static value.""" if not self.is_connected and ( - self.required == Requiredness.OPTIONAL or - self.required == Requiredness.REQUIRED_IF_CONNECTED): + self.required == Requiredness.OPTIONAL or self.required == Requiredness.REQUIRED_IF_CONNECTED + ): return self._value if self._input_mode == InputMode.QUEUE: - return self._queue.get() + return self.queue.get() elif self._input_mode == InputMode.STICKY: - if not self._queue.empty() or self._value is None: - value = self._queue.get() + if not self.queue.empty() or self._value is None: + value = self.queue.get() if not is_EOF(value): # Ignore EOFs on sticky inputs, only queue inputs matter for termination self._value = value @@ -153,6 +172,23 @@ def ref_count(self) -> int: """Get the reference count of the input.""" return self._ref_count + def apply_config(self, config: InputConfig): + """Apply config from the flyde flow to the input.""" + self._value = config.value + if config.type == InputType.DYNAMIC: + self._input_mode = InputMode.QUEUE + else: + self._input_mode = InputMode.STICKY + + # Apply Python type hint based on supported config type + if config.type != InputType.DYNAMIC and self.type is None: + self.type = { + InputType.NUMBER: int, + InputType.BOOLEAN: bool, + InputType.JSON: dict, + InputType.STRING: str, + }[config.type] + class Output: """Output is an interface for setting output data for a component.""" @@ -197,9 +233,7 @@ def connected(self) -> bool: def send(self, value: Any): """Put a value in the output queue.""" if self.type is not None and not is_EOF(value) and not isinstance(value, self.type): # type: ignore - raise ValueError( - f'Output "{self.id}": value {value} is not of type {self.type}' - ) + raise ValueError(f'Output "{self.id}": value {value} is not of type {self.type}') if len(self._queues) == 0: raise ValueError(f'Output "{self.id}": has no connected queues') @@ -294,11 +328,11 @@ def __init__( def inc_ref_count(self): # Need to increase ref count of the RedriveQueue - self._queue.inc_ref_count() # type: ignore + self._queue.inc_ref_count() # type: ignore return super().inc_ref_count() def dec_ref_count(self): - self._queue.dec_ref_count() # type: ignore + self._queue.dec_ref_count() # type: ignore return super().dec_ref_count() diff --git a/flyde/io.pyi b/flyde/io.pyi index 9a35e0d..e6f47d3 100644 --- a/flyde/io.pyi +++ b/flyde/io.pyi @@ -1,4 +1,5 @@ from _typeshed import Incomplete +from dataclasses import dataclass from enum import Enum from queue import Queue from typing import Any @@ -8,6 +9,21 @@ EOF: Incomplete def is_EOF(value: Any) -> bool: """Checks if a value is an EOF signal.""" +class InputType(Enum): + """Input type contains all input types supported by Flyde.""" + DYNAMIC = 'dynamic' + NUMBER = 'number' + BOOLEAN = 'boolean' + JSON = 'json' + STRING = 'string' + +@dataclass +class InputConfig: + """Configuration of an input in a Flyde flow.""" + type: InputType + value: Any + def __init__(self, type, value) -> None: ... + class InputMode(Enum): """InputMode is the mode of an input. @@ -87,6 +103,8 @@ class Input: @property def ref_count(self) -> int: """Get the reference count of the input.""" + def apply_config(self, config: InputConfig): + """Apply config from the flyde flow to the input.""" class Output: """Output is an interface for setting output data for a component.""" diff --git a/flyde/node.py b/flyde/node.py index d01545e..2821236 100644 --- a/flyde/node.py +++ b/flyde/node.py @@ -1,19 +1,73 @@ import logging from abc import ABC, abstractmethod from copy import deepcopy +from dataclasses import dataclass +from enum import Enum from threading import Event, Lock, Thread from typing import Any, Callable from uuid import uuid4 -from flyde.io import GraphPort, InputMode, Input, Output, EOF, Requiredness, is_EOF, Connection +from flyde.io import EOF, Connection, GraphPort, Input, InputConfig, InputMode, Output, Requiredness, is_EOF logger = logging.getLogger(__name__) SUPPORTED_MACROS = ["InlineValue", "Conditional", "GetAttribute"] + +class InstanceType(Enum): + """InstanceType is the type of an instance. + + VISUAL: The instance is a visual node. + CODE: The instance is a code node. + """ + + VISUAL = "visual" + CODE = "code" + + +class InstanceSourceType(Enum): + """InstanceSourceType is the source type of an instance. + + FILE: The instance is created from a file. + PACKAGE: The instance is created from a built in package.""" + + FILE = "file" + PACKAGE = "package" + + +@dataclass +class InstanceSource: + """Source configuration of an instance.""" + + type: InstanceSourceType + data: str + + +@dataclass +class InstanceArgs: + """Arguments to pass to the instance factory.""" + + id: str + display_name: str + stopped: Event | None + config: dict[str, InputConfig] + macro_data: dict[str, Any] | None = None + type: InstanceType = InstanceType.CODE + source: InstanceSource | None = None + + def to_dict(self) -> dict: + """Convert the instance arguments to a dictionary.""" + return { + "id": self.id, + "display_name": self.display_name, + "stopped": self.stopped, + "config": self.config, + } + + # InstanceFactory is a function that creates a new instance of a node. # It can create instances dynamically based on the node ID. -InstanceFactory = Callable[[str, dict], Any] +InstanceFactory = Callable[[str, InstanceArgs], Any] class Node(ABC): @@ -22,7 +76,7 @@ class Node(ABC): Attributes: id (str): A unique identifier for the node. node_type (str): The node type identifier. - input_config (dict): A dictionary of input pin configurations. + config (dict): A dictionary of input pin configurations. display_name (str): A human-readable name for the node. inputs (dict[str, Input]): Node input map. outputs (dict[str, Output]): Node output map. @@ -36,17 +90,17 @@ def __init__( /, id: str, node_type: str = "", - input_config: dict[str, InputMode] = {}, display_name: str = "", inputs: dict[str, Input] = {}, outputs: dict[str, Output] = {}, stopped: Event = Event(), + config: dict[str, InputConfig] = {}, ): node_type = node_type if node_type else self.__class__.__name__ self._node_type = node_type self._id = id if id else create_instance_id(node_type) - self._input_config = input_config self._display_name = display_name if display_name else node_type + self._config = config if len(inputs) > 0: self.inputs = inputs @@ -58,6 +112,8 @@ def __init__( for k, v in self.inputs.items(): v.id = f"{self._id}.{k}" + if k in self._config: + v.apply_config(self._config[k]) if len(outputs) > 0: self.outputs = outputs @@ -121,22 +177,30 @@ def from_yaml(cls, create: InstanceFactory, yml: dict): node_class_name = yml.get("nodeId", "VisualNode") if node_class_name == "VisualNode": return Graph.from_yaml(create, yml) - args = { - "id": yml["id"], - "input_config": yml.get("inputConfig", {}), - "display_name": yml.get("displayName", ""), - "stopped": yml.get("stopped", None), # It's a hacky way to pass the stopped event to the constructor - } - # If macro parameters are present, pass them to the constructor - if "macroData" in yml: - args["macro_data"] = yml["macroData"] + + config = {k: InputConfig(**v) for k, v in yml.get("config", {}).items()} + + source = InstanceSource( + type=InstanceSourceType(yml.get("source", {}).get("type", "file").lower()), + data=yml.get("source", {}).get("data", ""), + ) + + args = InstanceArgs( + id=yml["id"], + display_name=yml.get("displayName", ""), + stopped=yml.get("stopped", None), # It's a hacky way to pass the stopped event to the constructor + config=config, + type=InstanceType(yml.get("type", "code").lower()), + source=source, + macro_data=yml.get("macroData", {}), + ) return create(node_class_name, args) def to_dict(self) -> dict: return { "id": self._id, "nodeId": self._node_type, - "inputConfig": self._input_config, + "config": self._config, "displayName": self._display_name, } @@ -156,18 +220,28 @@ def __init__(self, **kwargs): def run(self): if not hasattr(self, "process"): - raise NotImplementedError( - "Component does not have neither run() nor process() method. No code to run." - ) + raise NotImplementedError("Component does not have neither run() nor process() method. No code to run.") def worker(): logger.debug(f"Running {self._id} worker") + + # Check if all inputs are sticky or static (not queue) + # If so, we only run the loop once + all_sticky_or_static = True + for inp in self.inputs.values(): + if inp._input_mode == InputMode.QUEUE: + all_sticky_or_static = False + break + + run_once = len(self.inputs) > 0 and all_sticky_or_static + while not self._stop.is_set(): logger.debug(f"Waiting for inputs on {self._id}") inputs = {} queue_count = 0 queue_closed_count = 0 skip_iteration = False + for key, inp in self.inputs.items(): is_queue = inp._input_mode == InputMode.QUEUE value = inp.get() @@ -198,9 +272,7 @@ def worker(): logger.debug(f"Processing {self._id} with inputs: {inputs}") res = self.process(**inputs) # type: ignore - if isinstance(res, dict) or ( - isinstance(res, tuple) and hasattr(res, "_fields") - ): + if isinstance(res, dict) or (isinstance(res, tuple) and hasattr(res, "_fields")): # Send values to the outputs named as keys for k, v in res.items(): # type: ignore if k not in self.outputs: @@ -217,6 +289,11 @@ def worker(): if self.outputs[k].connected: self.outputs[k].send(v) + # If all inputs are sticky/static, exit after the first iteration + if run_once: + logger.debug(f"All inputs are sticky or static for {self._id}, stopping after first execution") + break + self.finish() logger.debug(f"Starting {self._id} thread") @@ -238,34 +315,20 @@ def to_ts(cls, name: str = "") -> str: if hasattr(cls, "inputs") and len(cls.inputs) > 0: inputs_str = ( "\n" - + ",\n".join( - [ - f' {k}: {{ description: "{v.description}" }}' - for k, v in cls.inputs.items() - ] - ) + + ",\n".join([f' {k}: {{ description: "{v.description}" }}' for k, v in cls.inputs.items()]) + "\n" ) outputs_str = "" if hasattr(cls, "outputs") and len(cls.outputs) > 0: outputs_str = ( "\n" - + ",\n".join( - [ - f' {k}: {{ description: "{v.description}" }}' - for k, v in cls.outputs.items() - ] - ) + + ",\n".join([f' {k}: {{ description: "{v.description}" }}' for k, v in cls.outputs.items()]) + "\n" ) safe_doc = "" if hasattr(cls, "__doc__") and cls.__doc__: - safe_doc = ( - cls.__doc__.replace("\n", "\\n") - .replace("\r", "\\r") - .replace('"', '\\"') - ) + safe_doc = cls.__doc__.replace("\n", "\\n").replace("\r", "\\r").replace('"', '\\"') return ( f"export const {name}: CodeNode = {{\n" @@ -286,7 +349,7 @@ def __init__( /, id: str = "", node_type: str = "", - input_config: dict[str, InputMode] = {}, + config: dict[str, InputConfig] = {}, display_name: str = "", instances: dict[str, Node] = {}, instances_stopped: dict[str, Event] = {}, @@ -298,7 +361,7 @@ def __init__( super().__init__( id=id, node_type=node_type, - input_config=input_config, + config=config, display_name=display_name, stopped=stopped, ) @@ -321,13 +384,13 @@ def __init__( if from_pin not in self.inputs: raise ValueError(f"Input {from_pin} not found in graph {self._id}") else: - self._check_pin('out', from_id, from_pin) + self._check_pin("out", from_id, from_pin) if to_id == "__this": if to_pin not in self.outputs: raise ValueError(f"Output {to_pin} not found in graph {self._id}") else: - self._check_pin('in', to_id, to_pin) + self._check_pin("in", to_id, to_pin) if from_id != "__this" and to_id != "__this": # Simple case: connect two instances inside the graph @@ -362,9 +425,7 @@ def _check_pin(self, pin_type: str, instance_id: str, pin_id: str): def run(self): """Run the graph.""" for instance in self._instances.values(): - logger.debug( - f"Running instance {instance._id} of type {instance._node_type}" - ) + logger.debug(f"Running instance {instance._id} of type {instance._node_type}") instance.run() def worker(): @@ -420,7 +481,7 @@ def from_yaml(cls, create: InstanceFactory, yml: dict): # Load metadata node_type = yml.get("nodeId", __name__) id = yml["id"] if "id" in yml else create_instance_id(node_type) - input_config = yml.get("inputConfig", {}) + config = {k: InputConfig(**v) for k, v in yml.get("config", {}).items()} display_name = yml.get("displayName", node_type) # Load instances and macros @@ -431,7 +492,7 @@ def from_yaml(cls, create: InstanceFactory, yml: dict): if "macroId" in ins: # Only InlineValue macros are supported for now if ins["macroId"] not in SUPPORTED_MACROS: - raise ValueError(f'Unsupported macro: {ins["macroId"]}') + raise ValueError(f"Unsupported macro: {ins['macroId']}") ins["nodeId"] = ins["macroId"] stopped = Event() ins["stopped"] = stopped @@ -441,9 +502,7 @@ def from_yaml(cls, create: InstanceFactory, yml: dict): logger.debug(f"Loaded instance {ins_id}") # Load connections and graph inputs/outputs - connections = [ - Connection.from_yaml(conn) for conn in yml.get("connections", []) - ] + connections = [Connection.from_yaml(conn) for conn in yml.get("connections", [])] inputs = {} for k, v in yml.get("inputs", {}).items(): if "mode" in v: @@ -462,7 +521,7 @@ def from_yaml(cls, create: InstanceFactory, yml: dict): return cls( id=id, node_type=node_type, - input_config=input_config, + config=config, display_name=display_name, instances=instances, instances_stopped=instances_stopped, @@ -477,7 +536,7 @@ def to_dict(self) -> dict: return { "id": self._id, "nodeId": self._node_type, - "inputConfig": self._input_config, + "config": self._config, "displayName": self._display_name, "inputs": self.inputs, "outputs": self.outputs, diff --git a/flyde/node.pyi b/flyde/node.pyi index 6fae1ec..0ef17dc 100644 --- a/flyde/node.pyi +++ b/flyde/node.pyi @@ -1,13 +1,53 @@ import abc from _typeshed import Incomplete from abc import ABC, abstractmethod -from flyde.io import Connection as Connection, EOF as EOF, GraphPort as GraphPort, Input as Input, InputMode as InputMode, Output as Output, Requiredness as Requiredness, is_EOF as is_EOF +from dataclasses import dataclass +from enum import Enum +from flyde.io import Connection as Connection, EOF as EOF, GraphPort as GraphPort, Input as Input, InputConfig as InputConfig, InputMode as InputMode, Output as Output, Requiredness as Requiredness, is_EOF as is_EOF from threading import Event from typing import Any, Callable logger: Incomplete SUPPORTED_MACROS: Incomplete -InstanceFactory = Callable[[str, dict], Any] + +class InstanceType(Enum): + """InstanceType is the type of an instance. + + VISUAL: The instance is a visual node. + CODE: The instance is a code node. + """ + VISUAL = 'visual' + CODE = 'code' + +class InstanceSourceType(Enum): + """InstanceSourceType is the source type of an instance. + + FILE: The instance is created from a file. + PACKAGE: The instance is created from a built in package.""" + FILE = 'file' + PACKAGE = 'package' + +@dataclass +class InstanceSource: + """Source configuration of an instance.""" + type: InstanceSourceType + data: str + def __init__(self, type, data) -> None: ... + +@dataclass +class InstanceArgs: + """Arguments to pass to the instance factory.""" + id: str + display_name: str + stopped: Event | None + config: dict[str, InputConfig] + macro_data: dict[str, Any] | None = ... + type: InstanceType = ... + source: InstanceSource | None = ... + def to_dict(self) -> dict: + """Convert the instance arguments to a dictionary.""" + def __init__(self, id, display_name, stopped, config, macro_data=..., type=..., source=...) -> None: ... +InstanceFactory = Callable[[str, InstanceArgs], Any] class Node(ABC, metaclass=abc.ABCMeta): """Node is the main building block of an application. @@ -15,7 +55,7 @@ class Node(ABC, metaclass=abc.ABCMeta): Attributes: id (str): A unique identifier for the node. node_type (str): The node type identifier. - input_config (dict): A dictionary of input pin configurations. + config (dict): A dictionary of input pin configurations. display_name (str): A human-readable name for the node. inputs (dict[str, Input]): Node input map. outputs (dict[str, Output]): Node output map. @@ -24,10 +64,10 @@ class Node(ABC, metaclass=abc.ABCMeta): outputs: dict[str, Output] _node_type: Incomplete _id: Incomplete - _input_config: Incomplete _display_name: Incomplete + _config: Incomplete _stopped: Incomplete - def __init__(self, /, id: str, node_type: str = '', input_config: dict[str, InputMode] = {}, display_name: str = '', inputs: dict[str, Input] = {}, outputs: dict[str, Output] = {}, stopped: Event = ...) -> None: ... + def __init__(self, /, id: str, node_type: str = '', display_name: str = '', inputs: dict[str, Input] = {}, outputs: dict[str, Output] = {}, stopped: Event = ..., config: dict[str, InputConfig] = {}) -> None: ... @abstractmethod def run(self): """Run the node. This method should be overridden by subclasses.""" @@ -68,7 +108,7 @@ class Graph(Node): _connections: Incomplete _instances: Incomplete _instances_stopped: Incomplete - def __init__(self, /, id: str = '', node_type: str = '', input_config: dict[str, InputMode] = {}, display_name: str = '', instances: dict[str, Node] = {}, instances_stopped: dict[str, Event] = {}, connections: list[Connection] = [], inputs: dict[str, GraphPort] = {}, outputs: dict[str, GraphPort] = {}, stopped: Event = ...) -> None: ... + def __init__(self, /, id: str = '', node_type: str = '', config: dict[str, InputConfig] = {}, display_name: str = '', instances: dict[str, Node] = {}, instances_stopped: dict[str, Event] = {}, connections: list[Connection] = [], inputs: dict[str, GraphPort] = {}, outputs: dict[str, GraphPort] = {}, stopped: Event = ...) -> None: ... def _check_pin(self, pin_type: str, instance_id: str, pin_id: str): """Check if the instance and pin exist.""" def run(self) -> None: diff --git a/pyproject.toml b/pyproject.toml index e820517..8fef1f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,4 +45,5 @@ dev = [ "mypy", "twine", "mkdocs", + "icecream", ] diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..f11cf63 --- /dev/null +++ b/ruff.toml @@ -0,0 +1 @@ +line-length = 120 diff --git a/tests/__init.py__ b/tests/__init__.py similarity index 100% rename from tests/__init.py__ rename to tests/__init__.py diff --git a/tests/test_io.py b/tests/test_io.py index bdebb89..61210ff 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -1,12 +1,15 @@ import unittest from queue import Queue + from flyde.io import ( - Input, - InputMode, - Output, EOF, Connection, ConnectionNode, + Input, + InputConfig, + InputMode, + InputType, + Output, Requiredness, ) @@ -275,6 +278,57 @@ def test_ref_count(self): input.dec_ref_count() self.assertEqual(input.ref_count, 0) + def test_apply_config(self): + test_cases = [ + { + "name": "dynamic input config", + "config": InputConfig(type=InputType.DYNAMIC, value=None), + "expected": {"value": None, "input_mode": InputMode.QUEUE, "type": None}, + }, + { + "name": "number input config", + "config": InputConfig(type=InputType.NUMBER, value=42), + "expected": {"value": 42, "input_mode": InputMode.STICKY, "type": int}, + }, + { + "name": "boolean input config", + "config": InputConfig(type=InputType.BOOLEAN, value=True), + "expected": {"value": True, "input_mode": InputMode.STICKY, "type": bool}, + }, + { + "name": "json input config", + "config": InputConfig(type=InputType.JSON, value={"key": "value"}), + "expected": {"value": {"key": "value"}, "input_mode": InputMode.STICKY, "type": dict}, + }, + { + "name": "string input config", + "config": InputConfig(type=InputType.STRING, value="test"), + "expected": {"value": "test", "input_mode": InputMode.STICKY, "type": str}, + }, + { + "name": "input config with preset type", + "config": InputConfig(type=InputType.NUMBER, value=42), + "preset_type": float, + "expected": {"value": 42, "input_mode": InputMode.STICKY, "type": float}, + }, + ] + + for test_case in test_cases: + with self.subTest(case=test_case["name"]): + # Create new input instance for each test + if "preset_type" in test_case: + input_inst = Input(type=test_case["preset_type"]) + else: + input_inst = Input() + + # Apply the config + input_inst.apply_config(test_case["config"]) + + # Check all expected values + self.assertEqual(input_inst._value, test_case["expected"]["value"]) + self.assertEqual(input_inst._input_mode, test_case["expected"]["input_mode"]) + self.assertEqual(input_inst.type, test_case["expected"]["type"]) + class TestOutput(unittest.TestCase): def setUp(self): diff --git a/tests/test_node.py b/tests/test_node.py index 02b967c..4fd5a75 100644 --- a/tests/test_node.py +++ b/tests/test_node.py @@ -1,9 +1,10 @@ import threading import unittest -from threading import Thread from queue import Queue -from flyde.io import Input, InputMode, Output, EOF -from flyde.node import Component +from threading import Thread + +from flyde.io import EOF, Input, InputMode, Output +from flyde.node import Component, InstanceArgs from tests.components import RepeatWordNTimes @@ -133,27 +134,21 @@ def expected_typescript(name): run: () => { return; }, }; -""".replace( - "{NAME}", name - ) +""".replace("{NAME}", name) - self.assertEqual( - RepeatWordNTimes.to_ts(), expected_typescript("RepeatWordNTimes") - ) - self.assertEqual( - RepeatWordNTimes.to_ts("RepeatWord"), expected_typescript("RepeatWord") - ) + self.assertEqual(RepeatWordNTimes.to_ts(), expected_typescript("RepeatWordNTimes")) + self.assertEqual(RepeatWordNTimes.to_ts("RepeatWord"), expected_typescript("RepeatWord")) def test_from_yaml(self): yaml = { "id": "repeat", "nodeId": "RepeatWordNTimes", - "inputConfig": {}, + "config": {}, "displayName": "Repeat", } - def factory(class_name: str, args: dict): - return RepeatWordNTimes(**args) + def factory(class_name: str, args: InstanceArgs): + return RepeatWordNTimes(**args.to_dict()) node = Component.from_yaml(factory, yaml) self.assertEqual(node._id, "repeat") @@ -165,18 +160,21 @@ def test_from_yaml_with_macrodata(self): yaml = { "id": "repeat", "nodeId": "RepeatWordNTimes", - "inputConfig": {}, + "config": {}, "displayName": "Repeat", "macroData": {"value": 100, "key": "foo"}, } - def factory(class_name: str, args: dict): - self.assertEqual(args["macro_data"]["value"], yaml["macroData"]["value"]) - self.assertEqual(args["macro_data"]["key"], yaml["macroData"]["key"]) + def factory(class_name: str, args: InstanceArgs): + self.assertIsNotNone(args.macro_data) + if args.macro_data is None: + raise ValueError("macro_data is None") + self.assertEqual(args.macro_data["value"], yaml["macroData"]["value"]) + self.assertEqual(args.macro_data["key"], yaml["macroData"]["key"]) # Drop macro_data from the args, otherwise there will be an exception # because the constructor doesn't support it. - del args["macro_data"] - return RepeatWordNTimes(**args) + args.macro_data = None + return RepeatWordNTimes(**args.to_dict()) node = Component.from_yaml(factory, yaml) self.assertEqual(node._id, "repeat") @@ -189,12 +187,76 @@ def test_to_dict(self): expected = { "id": "repeat", "nodeId": "RepeatWordNTimes", - "inputConfig": {}, + "config": {}, "displayName": "Repeat", } self.assertEqual(node.to_dict(), expected) +class AllStickyInputsComponent(Component): + """A component with only sticky inputs to test single execution.""" + + inputs = { + "a": Input(description="First sticky input", type=int, mode=InputMode.STICKY, value=5), + "b": Input(description="Second sticky input", type=int, mode=InputMode.STICKY, value=10), + } + + outputs = {"result": Output(description="Result of operation", type=int)} + + def process(self, a: int, b: int) -> dict[str, int]: + return {"result": a + b} + + +class TestAllStickyInputsComponent(unittest.TestCase): + def test_all_sticky_inputs_run_once(self): + """Test that a component with all sticky inputs runs only once.""" + node = AllStickyInputsComponent(id="sticky_test", display_name="Sticky Test") + + # Connect an output queue to capture results + out_q = Queue() + node.outputs["result"].connect(out_q) + + # Run the component + node.run() + + # Check that it produced a single result and then EOF + self.assertEqual(out_q.get(), 15) # 5 + 10 + self.assertEqual(out_q.get(), EOF) + + # Verify that the component has stopped + node.stopped.wait(timeout=1) + self.assertTrue(node.stopped.is_set(), "Component should have stopped after one execution") + + def test_all_sticky_inputs_with_config(self): + """Test that a component with all sticky inputs initializes from config and runs once.""" + from flyde.io import InputConfig, InputType + + # Create node with config values + node = AllStickyInputsComponent( + id="sticky_test", + display_name="Sticky Test", + config={ + "a": InputConfig(type=InputType.NUMBER, value=20), + "b": InputConfig(type=InputType.NUMBER, value=30), + }, + ) + + # Connect an output queue to capture results + out_q = Queue() + node.outputs["result"].connect(out_q) + + # Run the component + node.run() + + # Check that it produced a single result with the configured values and then EOF + self.assertEqual(out_q.get(), 50) # 20 + 30 + self.assertEqual(out_q.get(), EOF) + + # Verify that the component has stopped + node.stopped.wait(timeout=1) + self.assertTrue(node.stopped.is_set(), "Component should have stopped after one execution") + + class SourceComponent(Component): """A component that only has outputs.""" From 6c0c72ddda1f1cbe94ec4642e5f17b561b53faa4 Mon Sep 17 00:00:00 2001 From: Vladimir Sibirov Date: Sat, 10 May 2025 19:08:49 +0200 Subject: [PATCH 02/18] Update stdlib and fix tests, except for nested graphs --- flyde/flow.py | 10 ++ flyde/io.py | 2 +- flyde/io.pyi | 4 +- flyde/node.py | 29 +++-- flyde/node.pyi | 10 +- flyde/stdlib.py | 107 ++++++++-------- flyde/stdlib.pyi | 25 ++-- pyproject.toml | 3 +- tests/Repeat3Times.flyde | 52 +++++--- tests/TestFanIn.flyde | 64 +++++++--- tests/TestInOutFlow.flyde | 65 ++++++---- tests/TestIsolatedFlow.flyde | 27 ++-- tests/test_flow.py | 144 +++++++++++----------- tests/test_node.py | 14 +-- tests/test_stdlib.py | 233 +++++++++++++++++------------------ 15 files changed, 434 insertions(+), 355 deletions(-) diff --git a/flyde/flow.py b/flyde/flow.py index a0cde98..ea412d6 100644 --- a/flyde/flow.py +++ b/flyde/flow.py @@ -24,6 +24,9 @@ def __init__(self, imports: dict[str, list[str]]): self._graphs: dict[str, dict] = {} def _preload_imports(self, base_path: str, imports: dict[str, list[str]]): + if not imports: + return + for module, classes in imports.items(): logger.debug(f"Importing {module}") # If module name ends with .flyde it's a Graph @@ -57,6 +60,10 @@ def _load_graph(self, name: str, path: str): def _load_component(self, name: str, path: str): """Loads a component from a Python module.""" + # If component is already loaded, return + if name in self._components: + return + # Translate typescript file path to python module path = path.replace("/", ".").replace(".flyde.ts", "").replace("@", "") logger.debug(f"Importing module {path}") @@ -80,6 +87,9 @@ def create_component(self, name: str, args: InstanceArgs): raise ValueError(f"Component {name} does not have a valid source") self._load_component(name, args.source.data) + if name not in self._components: + raise ValueError(f"Component {name} could not be loaded") + # Create the component instance component = self._components[name] return component(**args.to_dict()) diff --git a/flyde/io.py b/flyde/io.py index d2e9a8e..c60f61b 100644 --- a/flyde/io.py +++ b/flyde/io.py @@ -28,7 +28,7 @@ class InputConfig: """Configuration of an input in a Flyde flow.""" type: InputType - value: Any + value: Optional[Any] = None class InputMode(Enum): diff --git a/flyde/io.pyi b/flyde/io.pyi index e6f47d3..b841fca 100644 --- a/flyde/io.pyi +++ b/flyde/io.pyi @@ -21,8 +21,8 @@ class InputType(Enum): class InputConfig: """Configuration of an input in a Flyde flow.""" type: InputType - value: Any - def __init__(self, type, value) -> None: ... + value: Any | None = ... + def __init__(self, type, value=...) -> None: ... class InputMode(Enum): """InputMode is the mode of an input. diff --git a/flyde/node.py b/flyde/node.py index 2821236..c2fdfdc 100644 --- a/flyde/node.py +++ b/flyde/node.py @@ -7,7 +7,7 @@ from typing import Any, Callable from uuid import uuid4 -from flyde.io import EOF, Connection, GraphPort, Input, InputConfig, InputMode, Output, Requiredness, is_EOF +from flyde.io import EOF, Connection, GraphPort, Input, InputConfig, InputMode, InputType, Output, Requiredness, is_EOF logger = logging.getLogger(__name__) @@ -50,8 +50,7 @@ class InstanceArgs: id: str display_name: str stopped: Event | None - config: dict[str, InputConfig] - macro_data: dict[str, Any] | None = None + config: dict[str, Any] type: InstanceType = InstanceType.CODE source: InstanceSource | None = None @@ -100,7 +99,8 @@ def __init__( self._node_type = node_type self._id = id if id else create_instance_id(node_type) self._display_name = display_name if display_name else node_type - self._config = config + self._config_raw = config or {} + self._config = self.parse_config(self._config_raw) if len(inputs) > 0: self.inputs = inputs @@ -128,6 +128,19 @@ def __init__( self._stopped = stopped + def parse_config(self, config: dict[str, Any]) -> dict[str, Any]: + """Parse the raw config into a typed config dictionary.""" + result = {} + for key, value in config.items(): + if isinstance(value, dict) and "type" in value and "value" in value: + result[key] = InputConfig( + type=InputType(value["type"]), + value=value["value"], + ) + else: + result[key] = value + return result + @abstractmethod def run(self): """Run the node. This method should be overridden by subclasses.""" @@ -178,7 +191,7 @@ def from_yaml(cls, create: InstanceFactory, yml: dict): if node_class_name == "VisualNode": return Graph.from_yaml(create, yml) - config = {k: InputConfig(**v) for k, v in yml.get("config", {}).items()} + config = yml.get("config", {}) source = InstanceSource( type=InstanceSourceType(yml.get("source", {}).get("type", "file").lower()), @@ -192,7 +205,6 @@ def from_yaml(cls, create: InstanceFactory, yml: dict): config=config, type=InstanceType(yml.get("type", "code").lower()), source=source, - macro_data=yml.get("macroData", {}), ) return create(node_class_name, args) @@ -489,11 +501,6 @@ def from_yaml(cls, create: InstanceFactory, yml: dict): instances_stopped = {} for ins in yml.get("instances", []): ins_id = ins["id"] - if "macroId" in ins: - # Only InlineValue macros are supported for now - if ins["macroId"] not in SUPPORTED_MACROS: - raise ValueError(f"Unsupported macro: {ins['macroId']}") - ins["nodeId"] = ins["macroId"] stopped = Event() ins["stopped"] = stopped logger.debug(f"Creating instance {ins_id}") diff --git a/flyde/node.pyi b/flyde/node.pyi index 0ef17dc..9c791b8 100644 --- a/flyde/node.pyi +++ b/flyde/node.pyi @@ -3,7 +3,7 @@ from _typeshed import Incomplete from abc import ABC, abstractmethod from dataclasses import dataclass from enum import Enum -from flyde.io import Connection as Connection, EOF as EOF, GraphPort as GraphPort, Input as Input, InputConfig as InputConfig, InputMode as InputMode, Output as Output, Requiredness as Requiredness, is_EOF as is_EOF +from flyde.io import Connection as Connection, EOF as EOF, GraphPort as GraphPort, Input as Input, InputConfig as InputConfig, InputMode as InputMode, InputType as InputType, Output as Output, Requiredness as Requiredness, is_EOF as is_EOF from threading import Event from typing import Any, Callable @@ -40,13 +40,12 @@ class InstanceArgs: id: str display_name: str stopped: Event | None - config: dict[str, InputConfig] - macro_data: dict[str, Any] | None = ... + config: dict[str, Any] type: InstanceType = ... source: InstanceSource | None = ... def to_dict(self) -> dict: """Convert the instance arguments to a dictionary.""" - def __init__(self, id, display_name, stopped, config, macro_data=..., type=..., source=...) -> None: ... + def __init__(self, id, display_name, stopped, config, type=..., source=...) -> None: ... InstanceFactory = Callable[[str, InstanceArgs], Any] class Node(ABC, metaclass=abc.ABCMeta): @@ -65,9 +64,12 @@ class Node(ABC, metaclass=abc.ABCMeta): _node_type: Incomplete _id: Incomplete _display_name: Incomplete + _config_raw: Incomplete _config: Incomplete _stopped: Incomplete def __init__(self, /, id: str, node_type: str = '', display_name: str = '', inputs: dict[str, Input] = {}, outputs: dict[str, Output] = {}, stopped: Event = ..., config: dict[str, InputConfig] = {}) -> None: ... + def parse_config(self, config: dict[str, Any]) -> dict[str, Any]: + """Parse the raw config into a typed config dictionary.""" @abstractmethod def run(self): """Run the node. This method should be overridden by subclasses.""" diff --git a/flyde/stdlib.py b/flyde/stdlib.py index c3d5c9c..f686f14 100644 --- a/flyde/stdlib.py +++ b/flyde/stdlib.py @@ -1,9 +1,10 @@ import re +from dataclasses import dataclass from enum import Enum -from typing import Any +from typing import Any, Union +from flyde.io import Input, InputConfig, InputMode, InputType, Output from flyde.node import Component -from flyde.io import Input, Output, InputMode class InlineValue(Component): @@ -11,11 +12,11 @@ class InlineValue(Component): outputs = {"value": Output(description="The constant value")} - def __init__(self, macro_data: dict, **kwargs): + def __init__(self, **kwargs): super().__init__(**kwargs) - if "value" in macro_data: - value = macro_data["value"] - self.value = self._get_inline_value(value) if self._is_inline_dict(value) else value + if "value" in self._config: + value = self._config["value"] + self.value = value.value else: raise ValueError("Missing value in InlineValue configuration.") @@ -24,15 +25,6 @@ def process(self): # Inline value only runs once self.stop() - def _is_inline_dict(self, value: Any) -> bool: - """Check if a value is an inline Flyde value dict, which has `type` and `value` keys.""" - supported_inline_types = ["dynamic", "string", "number", "boolean", "json", "select", "longtext"] - return isinstance(value, dict) and "type" in value and value["type"] in supported_inline_types - - def _get_inline_value(self, value: Any) -> Any: - """Get the value from an inline Flyde value output.""" - return value["value"] - class _ConditionType(Enum): """Condition type enumeration.""" @@ -46,30 +38,29 @@ class _ConditionType(Enum): NotExists = "NOT_EXISTS" +@dataclass +class _ConditionConfig: + """Configuration etry for the condition type.""" + + type: _ConditionType + + class _ConditionalConfig: """Conditional configuration.""" - def __init__(self, yml: dict): - self.property_path = yml.get("propertyPath", "") + def __init__(self, config: dict[str, Union[InputConfig, _ConditionConfig]]): + if "condition" not in config: + raise ValueError("Missing 'condition' in Conditional configuration.") + if not isinstance(config["condition"], _ConditionConfig): + raise ValueError("Invalid 'condition' in Conditional configuration.") + condition = config["condition"] + self.condition_type = _ConditionType(condition.type) - condition = yml.get("condition", {}) - condition_type = condition.get("type", "EQUAL") - try: - self.condition_type = _ConditionType(condition_type) - except ValueError: - raise ValueError(f"Unsupported condition type: {condition_type}") - self.condition_data = condition.get("data", "") + if "leftOperand" in config and isinstance(config["leftOperand"], InputConfig): + self.left_operand: InputConfig = config["leftOperand"] - left_operand = yml.get("leftOperand", {}) - self.left_operand = { - "type": left_operand.get("type", "dynamic"), - "value": left_operand.get("value", ""), - } - right_operand = yml.get("rightOperand", {}) - self.right_operand = { - "type": right_operand.get("type", "dynamic"), - "value": right_operand.get("value", ""), - } + if "rightOperand" in config and isinstance(config["rightOperand"], InputConfig): + self.right_operand = config["rightOperand"] class Conditional(Component): @@ -84,15 +75,25 @@ class Conditional(Component): "false": Output(description="Output when the condition is false"), } - def __init__(self, macro_data: dict, **kwargs): + def parse_config(self, config: dict[str, Any]) -> dict[str, Any]: + """Parse the raw config, handling the 'condition' special case.""" + result = super().parse_config(config) # type: ignore + + # Handle the condition special case + if "condition" in result and isinstance(result["condition"], dict) and "type" in result["condition"]: + result["condition"] = _ConditionConfig(**result["condition"]) + + return result + + def __init__(self, **kwargs): super().__init__(**kwargs) - self._config = _ConditionalConfig(macro_data) - if self._config.left_operand["type"] != "dynamic": + self._config = _ConditionalConfig(self._config) + if hasattr(self._config, "left_operand") and self._config.left_operand.type != InputType.DYNAMIC: self.inputs["leftOperand"]._input_mode = InputMode.STATIC - self.inputs["leftOperand"].value = self._config.left_operand["value"] - if self._config.right_operand["type"] != "dynamic": + self.inputs["leftOperand"].value = self._config.left_operand.value + if hasattr(self._config, "right_operand") and self._config.right_operand.type != InputType.DYNAMIC: self.inputs["rightOperand"]._input_mode = InputMode.STATIC - self.inputs["rightOperand"].value = self._config.right_operand["value"] + self.inputs["rightOperand"].value = self._config.right_operand.value def _evaluate(self, left_operand: Any, right_operand: Any) -> bool: condition_type = self._config.condition_type @@ -132,22 +133,20 @@ class GetAttribute(Component): } outputs = {"value": Output(description="The attribute value")} - def __init__(self, macro_data: dict, **kwargs): + def __init__(self, **kwargs): super().__init__(**kwargs) - if "key" not in macro_data: + if "key" not in self._config: raise ValueError("Missing 'key' in GetAttribute configuration.") - key = macro_data["key"] - self.value = None - if "value" in key: - self.value = key["value"] - if "type" in key: - if key["type"] == "static": - self.inputs["key"]._input_mode = InputMode.STATIC # type: ignore - self.inputs["key"].value = self.value - else: - self.inputs["key"]._input_mode = InputMode.STICKY # type: ignore - if self.value is not None: - self.inputs["key"].value = self.value + key = self._config["key"] + if not isinstance(key, InputConfig): + raise ValueError("Invalid 'key' in GetAttribute configuration.") + if key.type == InputType.DYNAMIC: + self.inputs["key"]._input_mode = InputMode.STICKY # type: ignore + if key.value is not None: + self.inputs["key"].value = key.value + else: + self.inputs["key"]._input_mode = InputMode.STATIC # type: ignore + self.inputs["key"].value = key.value def process(self, object: Any, key: str): keys = key.split(".") diff --git a/flyde/stdlib.pyi b/flyde/stdlib.pyi index fe02ae1..88d1ef5 100644 --- a/flyde/stdlib.pyi +++ b/flyde/stdlib.pyi @@ -1,6 +1,7 @@ from _typeshed import Incomplete +from dataclasses import dataclass from enum import Enum -from flyde.io import Input as Input, InputMode as InputMode, Output as Output +from flyde.io import Input as Input, InputConfig as InputConfig, InputMode as InputMode, InputType as InputType, Output as Output from flyde.node import Component as Component from typing import Any @@ -8,12 +9,8 @@ class InlineValue(Component): """InlineValue sends a constant value to output.""" outputs: Incomplete value: Incomplete - def __init__(self, macro_data: dict, **kwargs) -> None: ... + def __init__(self, **kwargs) -> None: ... def process(self) -> None: ... - def _is_inline_dict(self, value: Any) -> bool: - """Check if a value is an inline Flyde value dict, which has `type` and `value` keys.""" - def _get_inline_value(self, value: Any) -> Any: - """Get the value from an inline Flyde value output.""" class _ConditionType(Enum): """Condition type enumeration.""" @@ -25,21 +22,27 @@ class _ConditionType(Enum): Exists = 'EXISTS' NotExists = 'NOT_EXISTS' +@dataclass +class _ConditionConfig: + """Configuration etry for the condition type.""" + type: _ConditionType + def __init__(self, type) -> None: ... + class _ConditionalConfig: """Conditional configuration.""" - property_path: Incomplete condition_type: Incomplete - condition_data: Incomplete left_operand: Incomplete right_operand: Incomplete - def __init__(self, yml: dict) -> None: ... + def __init__(self, config: dict[str, InputConfig | _ConditionConfig]) -> None: ... class Conditional(Component): """Conditional component evaluates a condition against the input and sends the result to output.""" inputs: Incomplete outputs: Incomplete + def parse_config(self, config: dict[str, Any]) -> dict[str, Any]: + """Parse the raw config, handling the 'condition' special case.""" _config: Incomplete - def __init__(self, macro_data: dict, **kwargs) -> None: ... + def __init__(self, **kwargs) -> None: ... def _evaluate(self, left_operand: Any, right_operand: Any) -> bool: ... def process(self, leftOperand: Any, rightOperand: Any): ... @@ -48,5 +51,5 @@ class GetAttribute(Component): inputs: Incomplete outputs: Incomplete value: Incomplete - def __init__(self, macro_data: dict, **kwargs) -> None: ... + def __init__(self, **kwargs) -> None: ... def process(self, object: Any, key: str): ... diff --git a/pyproject.toml b/pyproject.toml index 8fef1f2..a7e022e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pyflyde" -version = "0.0.12" +version = "0.1.0-alpha" requires-python = ">= 3.9" authors = [{ name = "Vladimir Sibirov" }] description = "Python SDK and runtime for Flyde - a visual flow-based programming language and IDE." @@ -18,6 +18,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Operating System :: OS Independent", "Topic :: Scientific/Engineering", "Topic :: Software Development :: Libraries", diff --git a/tests/Repeat3Times.flyde b/tests/Repeat3Times.flyde index 91c84ee..4adf691 100644 --- a/tests/Repeat3Times.flyde +++ b/tests/Repeat3Times.flyde @@ -1,9 +1,4 @@ -imports: - "@flyde/stdlib": - - InlineValue - components.flyde.ts: - - RepeatWordNTimes - - Capitalize +imports: {} node: instances: - pos: @@ -14,24 +9,45 @@ node: times: mode: sticky nodeId: RepeatWordNTimes + config: + word: + type: dynamic + value: "{{word}}" + times: + type: dynamic + value: "{{times}}" + type: code + source: + type: file + data: components.flyde.ts - pos: - x: 13.582794189453125 - y: -132.9213885498047 + x: -422.4172058105469 + y: 126.0786114501953 id: inl-ytsuyrje4syeb4qduymsfkl2 inputConfig: {} - visibleInputs: [] - nodeId: InlineValue__inl-ytsuyrje4syeb4qduymsfkl2 - macroId: InlineValue - macroData: + nodeId: InlineValue + config: value: type: number value: 3 + type: code + source: + type: package + data: "@flyde/stdlib" - pos: - x: -68.75665283203125 - y: 194.59005737304688 + x: 109.24334716796875 + y: 55.590057373046875 id: Capitalize-790499u inputConfig: {} nodeId: Capitalize + config: + inp: + type: dynamic + value: "{{inp}}" + type: code + source: + type: file + data: components.flyde.ts connections: - from: insId: inl-ytsuyrje4syeb4qduymsfkl2 @@ -66,13 +82,13 @@ node: delayed: false inputsPosition: word: - x: -92.08137329101562 - y: -227.9133740234375 + x: -351.0813732910156 + y: 52.08662597656249 outputsPosition: result: x: -23.264428942324532 y: 237.25953921502617 word3x: - x: -40.05281249999999 - y: 369.2831286621094 + x: 345.94718750000004 + y: 57.28312866210939 description: For each input string, sends a string with the same conent repeated 3 times diff --git a/tests/TestFanIn.flyde b/tests/TestFanIn.flyde index 25f9cad..31397a8 100644 --- a/tests/TestFanIn.flyde +++ b/tests/TestFanIn.flyde @@ -1,10 +1,4 @@ -imports: - "@flyde/stdlib": - - InlineValue - components.flyde.ts: - - Format - - Capitalize - - Echo +imports: {} node: instances: - pos: @@ -13,32 +7,62 @@ node: id: Format-4s04bag inputConfig: {} nodeId: Format + config: + inp: + type: dynamic + value: "{{inp}}" + format: + type: dynamic + value: "{{format}}" + type: code + source: + type: file + data: components.flyde.ts - pos: - x: -64.1663818359375 - y: 56.65972900390625 + x: -276.1663818359375 + y: -58.34027099609375 id: Capitalize-ch04buf inputConfig: {} nodeId: Capitalize + config: + inp: + type: dynamic + value: "{{inp}}" + type: code + source: + type: file + data: components.flyde.ts - pos: - x: -148.65975585937497 - y: 238.9669189453125 + x: -5.659755859374968 + y: 54.9669189453125 id: Echo-eb14b06 inputConfig: {} nodeId: Echo + config: + inp: + type: dynamic + value: "{{inp}}" + type: code + source: + type: file + data: components.flyde.ts - pos: - x: -328.1567211914063 - y: -90.1646710205078 + x: -594.1567211914063 + y: 120.8353289794922 id: xzi4aah4ewf1iw7q4a3jz9oj inputConfig: {} - nodeId: InlineValue__xzi4aah4ewf1iw7q4a3jz9oj - macroId: InlineValue - macroData: + nodeId: InlineValue + config: type: type: string value: string value: type: string value: Hello, {inp}! + type: code + source: + type: package + data: "@flyde/stdlib" connections: - from: insId: Format-4s04bag @@ -91,12 +115,12 @@ node: delayed: false inputsPosition: str: - x: -119.99874877929688 - y: -99.46359436035156 + x: -606.9987487792969 + y: -29.463594360351564 outputsPosition: result: x: -23.264428942324532 y: 237.25953921502617 out: - x: -120.79533203125001 - y: 390.6666717529297 + x: 223.20466796875 + y: 54.66667175292969 diff --git a/tests/TestInOutFlow.flyde b/tests/TestInOutFlow.flyde index 0e55838..edb9d63 100644 --- a/tests/TestInOutFlow.flyde +++ b/tests/TestInOutFlow.flyde @@ -1,26 +1,27 @@ -imports: - "@flyde/stdlib": - - Conditional - - InlineValue - components.flyde.ts: - - Echo - - Format +imports: {} node: instances: - pos: - x: -79.91999999999999 - y: 47.5 + x: 244.08 + y: -87.5 id: Echo-h3049mb inputConfig: {} nodeId: Echo + config: + inp: + type: dynamic + value: "{{inp}}" + type: code + source: + type: file + data: components.flyde.ts - pos: x: -17.077154541015602 y: -87.20937501999856 id: ppsa1z6ja2w6yyo0sig7hvww inputConfig: {} - nodeId: Conditional__ppsa1z6ja2w6yyo0sig7hvww - macroId: Conditional - macroData: + nodeId: Conditional + config: leftOperand: type: dynamic value: "{{value}}" @@ -29,20 +30,34 @@ node: type: string condition: type: EXISTS + type: code + source: + type: package + data: "@flyde/stdlib" - pos: - x: 96.42891601562502 - y: 48.46386716750146 + x: 241.42891601562496 + y: 13.463867167501462 id: Format-ve0397r inputConfig: {} nodeId: Format + config: + inp: + type: dynamic + value: "{{inp}}" + format: + type: dynamic + value: "{{format}}" + type: code + source: + type: file + data: components.flyde.ts - pos: - x: 227.21648071289064 - y: -86.37317262742044 + x: -32.78351928710936 + y: 33.626827372579555 id: apqbu37rhnui31o8qaud8ek8 inputConfig: {} - nodeId: InlineValue__apqbu37rhnui31o8qaud8ek8 - macroId: InlineValue - macroData: + nodeId: InlineValue + config: type: type: string value: string @@ -52,6 +67,10 @@ node: label: type: string value: '"ERR: msg is empty"' + type: code + source: + type: package + data: "@flyde/stdlib" connections: - from: insId: Echo-h3049mb @@ -98,12 +117,12 @@ node: delayed: false inputsPosition: inMsg: - x: 35.251495361328125 - y: -151.84883300781252 + x: -184.74850463867188 + y: -75.84883300781252 outputsPosition: result: x: -23.264428942324532 y: 237.25953921502617 outMsg: - x: -70.891923828125 - y: 220.63164794921875 + x: 631.1080761718749 + y: -93.36835205078125 diff --git a/tests/TestIsolatedFlow.flyde b/tests/TestIsolatedFlow.flyde index b856618..12777e1 100644 --- a/tests/TestIsolatedFlow.flyde +++ b/tests/TestIsolatedFlow.flyde @@ -1,27 +1,34 @@ -imports: - "@flyde/stdlib": - - InlineValue - components.flyde.ts: - - Echo +imports: {} node: instances: - pos: - x: -95.34318359375001 - y: -154.18736450195314 + x: -336.34318359375004 + y: 59.81263549804686 id: qxavnllf1foh9ivxvtz2hkad inputConfig: {} - nodeId: InlineValue__qxavnllf1foh9ivxvtz2hkad - macroId: InlineValue - macroData: + nodeId: InlineValue + config: value: type: string value: Hello + type: code + source: + type: package + data: "@flyde/stdlib" - pos: x: -83.18318359374999 y: 60.36263549804687 id: Echo-x8049tw inputConfig: {} nodeId: Echo + config: + inp: + type: dynamic + value: "{{inp}}" + type: code + source: + type: file + data: components.flyde.ts connections: - from: insId: qxavnllf1foh9ivxvtz2hkad diff --git a/tests/test_flow.py b/tests/test_flow.py index a855f65..3cb887e 100644 --- a/tests/test_flow.py +++ b/tests/test_flow.py @@ -1,7 +1,8 @@ -from queue import Queue import unittest -from flyde.io import EOF +from queue import Queue + from flyde.flow import Flow +from flyde.io import EOF class TestIsolatedFlow(unittest.TestCase): @@ -34,38 +35,38 @@ def test_flow(self): self.assertTrue(flow.stopped.is_set()) -class TestNestedFlow(unittest.TestCase): - def test_flow(self): - test_case = { - "inputs": { - "inp": ["Hello", "World", "!", EOF], - "n": [1, 2, EOF], - }, - "outputs": [ - "HELLOHELLOHELLO", - "WORLDWORLDWORLDWORLDWORLDWORLD", - "!!!!!!", - EOF, - ], - } - flow = Flow.from_file("tests/TestNestedFlow.flyde") - - inp_q = flow.node.inputs["inp"].queue - n_q = flow.node.inputs["n"].queue - out_q = Queue() - flow.node.outputs["out"].connect(out_q) - - flow.run() - - for i, inp in enumerate(test_case["inputs"]["inp"]): - inp_q.put(inp) - if i < len(test_case["inputs"]["n"]): - n_q.put(test_case["inputs"]["n"][i]) - out = out_q.get() - self.assertEqual(test_case["outputs"][i], out) - - flow.stopped.wait() - self.assertTrue(flow.stopped.is_set()) +# class TestNestedFlow(unittest.TestCase): +# def test_flow(self): +# test_case = { +# "inputs": { +# "inp": ["Hello", "World", "!", EOF], +# "n": [1, 2, EOF], +# }, +# "outputs": [ +# "HELLOHELLOHELLO", +# "WORLDWORLDWORLDWORLDWORLDWORLD", +# "!!!!!!", +# EOF, +# ], +# } +# flow = Flow.from_file("tests/TestNestedFlow.flyde") + +# inp_q = flow.node.inputs["inp"].queue +# n_q = flow.node.inputs["n"].queue +# out_q = Queue() +# flow.node.outputs["out"].connect(out_q) + +# flow.run() + +# for i, inp in enumerate(test_case["inputs"]["inp"]): +# inp_q.put(inp) +# if i < len(test_case["inputs"]["n"]): +# n_q.put(test_case["inputs"]["n"][i]) +# out = out_q.get() +# self.assertEqual(test_case["outputs"][i], out) + +# flow.stopped.wait() +# self.assertTrue(flow.stopped.is_set()) class TestFanInFlow(unittest.TestCase): @@ -103,41 +104,42 @@ def test_with_component(self): flow.stopped.wait() self.assertTrue(flow.stopped.is_set()) - def test_with_graph(self): - test_case = { - "inputs": ["John", EOF], - "outputs": [ - "JOHNJOHNJOHN", - "JOHNJOHNJOHN", - "HELLO, JOHN!HELLO, JOHN!HELLO, JOHN!", - EOF, - ], - } - flow = Flow.from_file("tests/TestFanInGraph.flyde") - in_q = flow.node.inputs["str"].queue - out_q = Queue() - flow.node.outputs["out"].connect(out_q) - - flow.run() - - for inp in test_case["inputs"]: - in_q.put(inp) - - # Get all outputs until EOF - output_list = [] - count = 0 - limit = len(test_case["outputs"]) - out = None - while count < limit and out != EOF: - out = out_q.get() - output_list.append(out) - count += 1 - - # Compare expected and actual lists ignoring the order of elements - self.assertCountEqual(test_case["outputs"], output_list) - # EOF must be the last output - self.assertEqual(EOF, output_list[-1]) - - flow.stopped.wait() - self.assertTrue(flow.stopped.is_set()) +# def test_with_graph(self): +# test_case = { +# "inputs": ["John", EOF], +# "outputs": [ +# "JOHNJOHNJOHN", +# "JOHNJOHNJOHN", +# "HELLO, JOHN!HELLO, JOHN!HELLO, JOHN!", +# EOF, +# ], +# } +# flow = Flow.from_file("tests/TestFanInGraph.flyde") + +# in_q = flow.node.inputs["str"].queue +# out_q = Queue() +# flow.node.outputs["out"].connect(out_q) + +# flow.run() + +# for inp in test_case["inputs"]: +# in_q.put(inp) + +# # Get all outputs until EOF +# output_list = [] +# count = 0 +# limit = len(test_case["outputs"]) +# out = None +# while count < limit and out != EOF: +# out = out_q.get() +# output_list.append(out) +# count += 1 + +# # Compare expected and actual lists ignoring the order of elements +# self.assertCountEqual(test_case["outputs"], output_list) +# # EOF must be the last output +# self.assertEqual(EOF, output_list[-1]) + +# flow.stopped.wait() +# self.assertTrue(flow.stopped.is_set()) diff --git a/tests/test_node.py b/tests/test_node.py index 4fd5a75..4313c8c 100644 --- a/tests/test_node.py +++ b/tests/test_node.py @@ -160,20 +160,16 @@ def test_from_yaml_with_macrodata(self): yaml = { "id": "repeat", "nodeId": "RepeatWordNTimes", - "config": {}, "displayName": "Repeat", - "macroData": {"value": 100, "key": "foo"}, + "config": {"value": 100, "key": "foo"}, } def factory(class_name: str, args: InstanceArgs): - self.assertIsNotNone(args.macro_data) - if args.macro_data is None: + self.assertIsNotNone(args.config) + if args.config is None: raise ValueError("macro_data is None") - self.assertEqual(args.macro_data["value"], yaml["macroData"]["value"]) - self.assertEqual(args.macro_data["key"], yaml["macroData"]["key"]) - # Drop macro_data from the args, otherwise there will be an exception - # because the constructor doesn't support it. - args.macro_data = None + self.assertEqual(args.config["value"], yaml["config"]["value"]) + self.assertEqual(args.config["key"], yaml["config"]["key"]) return RepeatWordNTimes(**args.to_dict()) node = Component.from_yaml(factory, yaml) diff --git a/tests/test_stdlib.py b/tests/test_stdlib.py index 670007c..43c1e4e 100644 --- a/tests/test_stdlib.py +++ b/tests/test_stdlib.py @@ -2,8 +2,8 @@ from queue import Queue from types import SimpleNamespace -from flyde.io import EOF -from flyde.stdlib import InlineValue, Conditional, GetAttribute +from flyde.io import EOF, InputConfig, InputType +from flyde.stdlib import Conditional, GetAttribute, InlineValue, _ConditionConfig, _ConditionType class TestInlineValue(unittest.TestCase): @@ -13,7 +13,7 @@ def test_inline_value(self): "outputs": {"value": "Hello"}, } out_q = Queue() - node = InlineValue(macro_data={"value": "Hello"}, id="test_inline_value") + node = InlineValue(id="test_inline_value", config={"value": InputConfig(type=InputType.STRING, value="Hello")}) node.outputs["value"].connect(out_q) node.run() self.assertEqual(test_case["outputs"]["value"], out_q.get()) @@ -26,10 +26,7 @@ def test_inline_value_dict(self): "outputs": {"value": "Hello"}, } out_q = Queue() - node = InlineValue( - macro_data={"value": {"type": "string", "value": "Hello"}}, - id="test_inline_value", - ) + node = InlineValue(id="test_inline_value", config={"value": InputConfig(type=InputType.STRING, value="Hello")}) node.outputs["value"].connect(out_q) node.run() self.assertEqual(test_case["outputs"]["value"], out_q.get()) @@ -42,17 +39,14 @@ def test_conditional(self): test_cases = [ { "name": "equal static string", - "yml": { - "leftOperand": { - "type": "static", - "value": "Apple", - }, - "rightOperand": { - "type": "dynamic", - }, - "condition": { - "type": "EQUAL", - }, + "config": { + "leftOperand": InputConfig(type=InputType.STRING, value="Apple"), + "rightOperand": InputConfig( + type=InputType.DYNAMIC, + ), + "condition": _ConditionConfig( + type=_ConditionType.Equal, + ), }, "inputs": { "leftOperand": [], @@ -66,16 +60,16 @@ def test_conditional(self): }, { "name": "not equal dynamic string", - "yml": { - "leftOperand": { - "type": "dynamic", - }, - "rightOperand": { - "type": "dynamic", - }, - "condition": { - "type": "NOT_EQUAL", - }, + "config": { + "leftOperand": InputConfig( + type=InputType.DYNAMIC, + ), + "rightOperand": InputConfig( + type=InputType.DYNAMIC, + ), + "condition": _ConditionConfig( + type=_ConditionType.NotEqual, + ), }, "inputs": { "leftOperand": ["Apple", "Banana", "apple", "Grape", EOF], @@ -88,17 +82,14 @@ def test_conditional(self): }, { "name": "contains static string", - "yml": { - "leftOperand": { - "type": "dynamic", - }, - "rightOperand": { - "type": "static", - "value": "Apple", - }, - "condition": { - "type": "CONTAINS", - }, + "config": { + "leftOperand": InputConfig( + type=InputType.DYNAMIC, + ), + "rightOperand": InputConfig(type=InputType.STRING, value="Apple"), + "condition": _ConditionConfig( + type=_ConditionType.Contains, + ), }, "inputs": { "leftOperand": [ @@ -117,17 +108,14 @@ def test_conditional(self): }, { "name": "not contains static string", - "yml": { - "leftOperand": { - "type": "dynamic", - }, - "rightOperand": { - "type": "static", - "value": "Apple", - }, - "condition": { - "type": "NOT_CONTAINS", - }, + "config": { + "leftOperand": InputConfig( + type=InputType.DYNAMIC, + ), + "rightOperand": InputConfig(type=InputType.STRING, value="Apple"), + "condition": _ConditionConfig( + type=_ConditionType.NotContains, + ), }, "inputs": { "leftOperand": [ @@ -146,17 +134,14 @@ def test_conditional(self): }, { "name": "regex matches static", - "yml": { - "leftOperand": { - "type": "dynamic", - }, - "rightOperand": { - "type": "static", - "value": "^[A-Z]", - }, - "condition": { - "type": "REGEX_MATCHES", - }, + "config": { + "leftOperand": InputConfig( + type=InputType.DYNAMIC, + ), + "rightOperand": InputConfig(type=InputType.STRING, value="^[A-Z]"), + "condition": _ConditionConfig( + type=_ConditionType.RegexMatches, + ), }, "inputs": { "leftOperand": [ @@ -176,17 +161,14 @@ def test_conditional(self): }, { "name": "exists", - "yml": { - "leftOperand": { - "type": "dynamic", - }, - "rightOperand": { - "type": "static", - "value": "this is not important", - }, - "condition": { - "type": "EXISTS", - }, + "config": { + "leftOperand": InputConfig( + type=InputType.DYNAMIC, + ), + "rightOperand": InputConfig(type=InputType.STRING, value="this is not important"), + "condition": _ConditionConfig( + type=_ConditionType.Exists, + ), }, "inputs": { "leftOperand": ["Apple", "", " ", " ", "banana", EOF], @@ -199,17 +181,14 @@ def test_conditional(self): }, { "name": "does not exist", - "yml": { - "leftOperand": { - "type": "dynamic", - }, - "rightOperand": { - "type": "static", - "value": "this is not important", - }, - "condition": { - "type": "NOT_EXISTS", - }, + "config": { + "leftOperand": InputConfig( + type=InputType.DYNAMIC, + ), + "rightOperand": InputConfig(type=InputType.STRING, value="this is not important"), + "condition": _ConditionConfig( + type=_ConditionType.NotExists, + ), }, "inputs": { "leftOperand": ["Apple", "", " ", " ", "banana", EOF], @@ -222,16 +201,14 @@ def test_conditional(self): }, { "name": "unsupported condition type", - "yml": { - "leftOperand": { - "type": "dynamic", - }, - "rightOperand": { - "type": "dynamic", - }, - "condition": { - "type": "UNSUPPORTED", - }, + "config": { + "leftOperand": InputConfig( + type=InputType.DYNAMIC, + ), + "rightOperand": InputConfig( + type=InputType.DYNAMIC, + ), + "condition": {"type": "UNSUPPORTED"}, # Will cause a ValueError in parse_config }, "inputs": { "leftOperand": ["Apple", "Banana", "apple", EOF], @@ -251,10 +228,10 @@ def test_conditional(self): if "raises" in test_case and test_case["raises"] is not None: with self.assertRaises(test_case["raises"]): - node = Conditional(test_case["yml"], id="test_conditional") + node = Conditional(id="test_conditional", config=test_case["config"]) continue - node = Conditional(test_case["yml"], id="test_conditional") + node = Conditional(id="test_conditional", config=test_case["config"]) left_q = node.inputs["leftOperand"].queue right_q = node.inputs["rightOperand"].queue node.outputs["true"].connect(true_q) @@ -288,9 +265,11 @@ def test_get_attribute(self): test_cases = [ { "name": "static attribute from a dict", - "key": { - "type": "static", - "value": "name", + "config": { + "key": InputConfig( + type=InputType.STRING, + value="name", + ), }, "inputs": { "object": [ @@ -305,9 +284,11 @@ def test_get_attribute(self): }, { "name": "sticky attribute from an object", - "key": { - "type": "sticky", - "value": "name", + "config": { + "key": InputConfig( + type=InputType.STRING, + value="name", + ), }, "inputs": { "object": [ @@ -316,13 +297,18 @@ def test_get_attribute(self): SimpleNamespace(nananan="Charlie"), EOF, ], - "key": ["name"], + "key": ["name", EOF], }, "outputs": ["Alice", "Bob", None, EOF], }, { "name": "dynamic attribute from a dict", - "key": {}, + "config": { + "key": InputConfig( + type=InputType.DYNAMIC, + value=None, + ), + }, "inputs": { "object": [ {"name": "Alice"}, @@ -335,10 +321,12 @@ def test_get_attribute(self): "outputs": ["Alice", "Bob", "Charlie", EOF], }, { - "name": "sticky attribute and non-object", - "key": { - "type": "sticky", - "value": "name", + "name": "static attribute and non-object", + "config": { + "key": InputConfig( + type=InputType.STRING, + value="name", + ), }, "inputs": { "object": [ @@ -347,15 +335,17 @@ def test_get_attribute(self): 123, EOF, ], - "key": ["name"], + "key": ["name", EOF], }, "outputs": ["Alice", None, None, EOF], }, { "name": "nested attribute with dot key notation", - "key": { - "type": "static", - "value": "address.city", + "config": { + "key": InputConfig( + type=InputType.STRING, + value="address.city", + ), }, "inputs": { "object": [ @@ -370,9 +360,11 @@ def test_get_attribute(self): }, { "name": "nested 3 levels deep", - "key": { - "type": "static", - "value": "address.city.zip", + "config": { + "key": InputConfig( + type=InputType.STRING, + value="address.city.zip", + ), }, "inputs": { "object": [ @@ -390,9 +382,9 @@ def test_get_attribute(self): for test_case in test_cases: attr_q = Queue() out_q = Queue() - node = GetAttribute( - macro_data={"key": test_case["key"]}, id="test_get_attribute" - ) + config = test_case["config"] + + node = GetAttribute(id="test_get_attribute", config=config) obj_q = node.inputs["object"].queue if len(test_case["inputs"]["key"]) > 0: attr_q = node.inputs["key"].queue @@ -400,8 +392,9 @@ def test_get_attribute(self): node.run() for i in range(len(test_case["inputs"]["object"])): obj_q.put(test_case["inputs"]["object"][i]) - if len(test_case["inputs"]["key"]) > 0 and i < len( - test_case["inputs"]["key"] - ): + if len(test_case["inputs"]["key"]) > 0 and i < len(test_case["inputs"]["key"]): attr_q.put(test_case["inputs"]["key"][i]) self.assertEqual(test_case["outputs"][i], out_q.get()) + + node.stopped.wait() + self.assertTrue(node.stopped.is_set()) From 8d08f52272e6bcf16df5c010c6f14f65f3e05ae5 Mon Sep 17 00:00:00 2001 From: Vladimir Sibirov Date: Sat, 10 May 2025 20:00:25 +0200 Subject: [PATCH 03/18] Fix Python 3.9 compatibility --- flyde/node.py | 6 +++--- flyde/stdlib.pyi | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/flyde/node.py b/flyde/node.py index c2fdfdc..a1bb75b 100644 --- a/flyde/node.py +++ b/flyde/node.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from enum import Enum from threading import Event, Lock, Thread -from typing import Any, Callable +from typing import Any, Callable, Optional from uuid import uuid4 from flyde.io import EOF, Connection, GraphPort, Input, InputConfig, InputMode, InputType, Output, Requiredness, is_EOF @@ -49,10 +49,10 @@ class InstanceArgs: id: str display_name: str - stopped: Event | None + stopped: Optional[Event] config: dict[str, Any] type: InstanceType = InstanceType.CODE - source: InstanceSource | None = None + source: Optional[InstanceSource] = None def to_dict(self) -> dict: """Convert the instance arguments to a dictionary.""" diff --git a/flyde/stdlib.pyi b/flyde/stdlib.pyi index 88d1ef5..321795b 100644 --- a/flyde/stdlib.pyi +++ b/flyde/stdlib.pyi @@ -50,6 +50,5 @@ class GetAttribute(Component): """Get an attribute from an object or dictionary.""" inputs: Incomplete outputs: Incomplete - value: Incomplete def __init__(self, **kwargs) -> None: ... def process(self, object: Any, key: str): ... From bc90ad5a81fc951cb8bc4ef80de0e4d539f5c16e Mon Sep 17 00:00:00 2001 From: Vladimir Sibirov Date: Sat, 10 May 2025 20:38:39 +0200 Subject: [PATCH 04/18] Fix the Clustering example --- examples/Clustering.flyde | 164 +++++++++++++++++++++++++++----------- flyde/node.py | 5 +- tests/test_node.py | 17 +++- 3 files changed, 137 insertions(+), 49 deletions(-) diff --git a/examples/Clustering.flyde b/examples/Clustering.flyde index b6129ec..3450f11 100644 --- a/examples/Clustering.flyde +++ b/examples/Clustering.flyde @@ -1,31 +1,27 @@ -imports: - "@flyde/stdlib": - - InlineValue - - GetAttribute - mylib/dataframe.flyde.ts: - - LoadDataset - - Scale - mylib/kmeans.flyde.ts: - - KMeansNClusters - - KMeansCluster - - PCA2 - - Visualize +imports: {} node: instances: - pos: - x: -44.67793457031249 - y: -40.51885223388672 + x: -688.3592700195312 + y: 60.69355010986328 id: LoadDataset-1a039uk inputConfig: {} nodeId: LoadDataset + config: + file_path: + type: dynamic + value: "{{file_path}}" + type: code + source: + type: file + data: mylib/dataframe.flyde.ts - pos: - x: -81.67130859374998 - y: -124.01171112060547 + x: -939.2435131835937 + y: 61.415870666503906 id: mc4t4fqqezd1gns4fb7ckxmc inputConfig: {} - nodeId: InlineValue__mc4t4fqqezd1gns4fb7ckxmc - macroId: InlineValue - macroData: + nodeId: InlineValue + config: type: type: string value: string @@ -35,32 +31,65 @@ node: label: type: string value: wine-clustering.csv + type: code + source: + type: package + data: "@flyde/stdlib" - pos: - x: -21.232984619140666 - y: 59.69538116455078 + x: -436.9881726074219 + y: 60.95093536376953 id: Scale-dz139az inputConfig: {} nodeId: Scale + config: + dataframe: + type: dynamic + value: "{{dataframe}}" + type: code + source: + type: file + data: mylib/dataframe.flyde.ts - pos: - x: 115.65844161987303 - y: 212.51250839233398 + x: -24.11493484497072 + y: 62.353389739990234 id: KMeansNClusters-5m2390u inputConfig: {} nodeId: KMeansNClusters + config: + scaled_dataframe: + type: dynamic + value: "{{scaled_dataframe}}" + max_clusters: + type: dynamic + value: "{{max_clusters}}" + type: code + source: + type: file + data: mylib/kmeans.flyde.ts - pos: - x: -21.067138671875 - y: 348.4979133605957 + x: 234.1243896484375 + y: 185.80514907836914 id: KMeansCluster-ql339i4 inputConfig: {} nodeId: KMeansCluster + config: + scaled_dataframe: + type: dynamic + value: "{{scaled_dataframe}}" + n_clusters: + type: dynamic + value: "{{n_clusters}}" + type: code + source: + type: file + data: mylib/kmeans.flyde.ts - pos: - x: 233.42520244598387 - y: 80.12830471992493 + x: -431.2221974563599 + y: -46.37674593925476 id: yquy5xqil6xp9tmb42ktmosp inputConfig: {} - nodeId: InlineValue__yquy5xqil6xp9tmb42ktmosp - macroId: InlineValue - macroData: + nodeId: InlineValue + config: value: type: number value: 20 @@ -70,42 +99,81 @@ node: label: type: string value: "20" + type: code + source: + type: package + data: "@flyde/stdlib" - pos: - x: -264.7014599609375 - y: 436.35204895019535 + x: 234.12208740234377 + y: 410.2254124450684 id: PCA2-mz439jk inputConfig: {} nodeId: PCA2 + config: + scaled_dataframe: + type: dynamic + value: "{{scaled_dataframe}}" + type: code + source: + type: file + data: mylib/kmeans.flyde.ts - pos: - x: -194.03480346679686 - y: 829.8976345825196 + x: 1010.514711303711 + y: 252.2306843566895 id: Visualize-yz539m4 inputConfig: {} nodeId: Visualize + config: + pca_components: + type: dynamic + value: "{{pca_components}}" + pca_centroids: + type: dynamic + value: "{{pca_centroids}}" + kmeans_result: + type: dynamic + value: "{{kmeans_result}}" + type: code + source: + type: file + data: mylib/kmeans.flyde.ts - pos: - x: -85.56712158203123 - y: 675.0107851791382 + x: 736.7720812988282 + y: 149.70720727920536 id: PCA2-e5639ct inputConfig: {} nodeId: PCA2 + config: + scaled_dataframe: + type: dynamic + value: "{{scaled_dataframe}}" + type: code + source: + type: file + data: mylib/kmeans.flyde.ts - pos: - x: -158.16623901367188 - y: 567.0884128027336 + x: 513.1246850585937 + y: 141.11043981841965 id: nmt2k6i80qpwu9fyuckbqefw inputConfig: {} - nodeId: GetAttribute__nmt2k6i80qpwu9fyuckbqefw - macroId: GetAttribute - macroData: + nodeId: GetAttribute + config: key: type: dynamic + object: + type: dynamic + value: "{{object}}" + type: code + source: + type: package + data: "@flyde/stdlib" - pos: - x: -94.66524780273448 - y: 458.4299138098137 + x: 234.12473022460927 + y: 310.8023236303703 id: xb5xo767x6u0ubdvvegi0e53 inputConfig: {} - nodeId: InlineValue__xb5xo767x6u0ubdvvegi0e53 - macroId: InlineValue - macroData: + nodeId: InlineValue + config: type: type: string value: string @@ -115,6 +183,10 @@ node: label: type: string value: '"centroids"' + type: code + source: + type: package + data: "@flyde/stdlib" connections: - from: insId: mc4t4fqqezd1gns4fb7ckxmc diff --git a/flyde/node.py b/flyde/node.py index a1bb75b..8073dcf 100644 --- a/flyde/node.py +++ b/flyde/node.py @@ -132,10 +132,11 @@ def parse_config(self, config: dict[str, Any]) -> dict[str, Any]: """Parse the raw config into a typed config dictionary.""" result = {} for key, value in config.items(): - if isinstance(value, dict) and "type" in value and "value" in value: + if isinstance(value, dict) and "type" in value and value["type"] in InputType: + config_value = value.get("value", None) result[key] = InputConfig( type=InputType(value["type"]), - value=value["value"], + value=config_value, ) else: result[key] = value diff --git a/tests/test_node.py b/tests/test_node.py index 4313c8c..4750e9f 100644 --- a/tests/test_node.py +++ b/tests/test_node.py @@ -3,7 +3,7 @@ from queue import Queue from threading import Thread -from flyde.io import EOF, Input, InputMode, Output +from flyde.io import EOF, Input, InputConfig, InputMode, InputType, Output from flyde.node import Component, InstanceArgs from tests.components import RepeatWordNTimes @@ -188,6 +188,21 @@ def test_to_dict(self): } self.assertEqual(node.to_dict(), expected) + def test_parse_config_with_type_only(self): + config = {"times": {"type": "number"}, "word": {"type": "string", "value": "default"}} + + node = RepeatWordNTimes(id="repeat", display_name="Repeat", config=config) + + self.assertIn("times", node._config) + self.assertIsInstance(node._config["times"], InputConfig) + self.assertEqual(node._config["times"].type, InputType.NUMBER) + self.assertIsNone(node._config["times"].value) + + self.assertIn("word", node._config) + self.assertIsInstance(node._config["word"], InputConfig) + self.assertEqual(node._config["word"].type, InputType.STRING) + self.assertEqual(node._config["word"].value, "default") + class AllStickyInputsComponent(Component): """A component with only sticky inputs to test single execution.""" From 6b48eda556e630e94bd3647ace9bee04673ebe9c Mon Sep 17 00:00:00 2001 From: Vladimir Sibirov Date: Sat, 10 May 2025 21:47:53 +0200 Subject: [PATCH 05/18] Python < 3.12 fix --- flyde/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flyde/node.py b/flyde/node.py index 8073dcf..4b6a146 100644 --- a/flyde/node.py +++ b/flyde/node.py @@ -132,7 +132,7 @@ def parse_config(self, config: dict[str, Any]) -> dict[str, Any]: """Parse the raw config into a typed config dictionary.""" result = {} for key, value in config.items(): - if isinstance(value, dict) and "type" in value and value["type"] in InputType: + if isinstance(value, dict) and "type" in value and value["type"] in [item.value for item in InputType]: config_value = value.get("value", None) result[key] = InputConfig( type=InputType(value["type"]), From 44a4875c0418476790a59fa5a5aa90886b56c219 Mon Sep 17 00:00:00 2001 From: Vladimir Sibirov Date: Sun, 11 May 2025 12:34:06 +0200 Subject: [PATCH 06/18] Implement the Http component in stdlib --- flyde/stdlib.py | 137 ++++++++++++++++++++++++- flyde/stdlib.pyi | 9 +- tests/test_stdlib.py | 236 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 378 insertions(+), 4 deletions(-) diff --git a/flyde/stdlib.py b/flyde/stdlib.py index f686f14..bfb4b6c 100644 --- a/flyde/stdlib.py +++ b/flyde/stdlib.py @@ -1,9 +1,11 @@ +import json import re from dataclasses import dataclass from enum import Enum -from typing import Any, Union +from typing import Any, Optional, Union +from urllib import error, parse, request -from flyde.io import Input, InputConfig, InputMode, InputType, Output +from flyde.io import Input, InputConfig, InputMode, InputType, Output, Requiredness from flyde.node import Component @@ -160,3 +162,134 @@ def process(self, object: Any, key: str): value = None break self.send("value", value) + + +class Http(Component): + """Http component makes HTTP requests with urllib.""" + + inputs = { + "url": Input(description="URL to request", required=Requiredness.REQUIRED), + "method": Input(description="HTTP method", type=str, required=Requiredness.REQUIRED), + "headers": Input(description="HTTP headers", type=dict, required=Requiredness.OPTIONAL), + "params": Input(description="URL parameters", type=dict, required=Requiredness.OPTIONAL), + "data": Input(description="Request body", type=dict, required=Requiredness.OPTIONAL), + } + outputs = { + "data": Output(description="Response data"), + } + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + if "method" in self._config and isinstance(self._config["method"], InputConfig): + if self._config["method"].type == InputType.DYNAMIC: + self.inputs["method"]._input_mode = InputMode.STICKY + else: + self.inputs["method"]._input_mode = InputMode.STATIC + self.inputs["method"].value = self._config["method"].value + else: + self.inputs["method"]._input_mode = InputMode.STATIC + self.inputs["method"].value = "GET" + + if "url" in self._config and isinstance(self._config["url"], InputConfig): + if self._config["url"].type == InputType.DYNAMIC: + self.inputs["url"]._input_mode = InputMode.QUEUE + else: + self.inputs["url"]._input_mode = InputMode.STATIC + self.inputs["url"].value = self._config["url"].value + + if "headers" in self._config and isinstance(self._config["headers"], InputConfig): + if self._config["headers"].type == InputType.DYNAMIC: + self.inputs["headers"]._input_mode = InputMode.STICKY + else: + self.inputs["headers"]._input_mode = InputMode.STATIC + self.inputs["headers"].value = self._config["headers"].value + else: + self.inputs["headers"]._input_mode = InputMode.STATIC + self.inputs["headers"].value = {} + + if "params" in self._config and isinstance(self._config["params"], InputConfig): + if self._config["params"].type == InputType.DYNAMIC: + self.inputs["params"]._input_mode = InputMode.STICKY + else: + self.inputs["params"]._input_mode = InputMode.STATIC + self.inputs["params"].value = self._config["params"].value + else: + self.inputs["params"]._input_mode = InputMode.STATIC + self.inputs["params"].value = {} + + if "data" in self._config and isinstance(self._config["data"], InputConfig): + if self._config["data"].type == InputType.DYNAMIC: + self.inputs["data"]._input_mode = InputMode.STICKY + else: + self.inputs["data"]._input_mode = InputMode.STATIC + self.inputs["data"].value = self._config["data"].value + else: + self.inputs["data"]._input_mode = InputMode.STATIC + self.inputs["data"].value = {} + + def process( + self, + url: str, + method: str, + headers: Optional[dict] = None, + params: Optional[dict] = None, + data: Optional[dict] = None, + ): + try: + if params: + url_parts = list(parse.urlparse(url)) + query = dict(parse.parse_qsl(url_parts[4])) + query.update(params) + url_parts[4] = parse.urlencode(query) + url = parse.urlunparse(url_parts) + + req = request.Request(url) + req.method = method + + if headers: + for key, value in headers.items(): + req.add_header(key, value) + + data_bytes = None + if data and method != "GET": + data_bytes = json.dumps(data).encode("utf-8") + req.add_header("Content-Type", "application/json") + req.add_header("Content-Length", str(len(data_bytes))) + + with request.urlopen(req, data=data_bytes) as response: + content_type = response.headers.get('Content-Type', '') + response_data = response.read() + + # Handle text-based responses + if 'text/' in content_type or 'json' in content_type or 'xml' in content_type or 'application/javascript' in content_type: + # Extract charset from content-type header if present + charset = 'utf-8' # Default charset + if 'charset=' in content_type: + charset_part = content_type.split('charset=')[1] + if ';' in charset_part: + charset = charset_part.split(';')[0].strip() + else: + charset = charset_part.strip() + + try: + response_data = response_data.decode(charset) + except (UnicodeDecodeError, LookupError): + # Fallback to utf-8 if specified charset fails + response_data = response_data.decode('utf-8', errors='replace') + + # Try to parse JSON if the content type indicates JSON + if 'json' in content_type: + try: + response_data = json.loads(response_data) + except json.JSONDecodeError: + pass + # Binary data remains as bytes + + self.send("data", response_data) + except error.HTTPError as e: + raise e + except error.URLError as e: + raise e + except Exception as e: + raise e diff --git a/flyde/stdlib.pyi b/flyde/stdlib.pyi index 321795b..88eb9c7 100644 --- a/flyde/stdlib.pyi +++ b/flyde/stdlib.pyi @@ -1,7 +1,7 @@ from _typeshed import Incomplete from dataclasses import dataclass from enum import Enum -from flyde.io import Input as Input, InputConfig as InputConfig, InputMode as InputMode, InputType as InputType, Output as Output +from flyde.io import Input as Input, InputConfig as InputConfig, InputMode as InputMode, InputType as InputType, Output as Output, Requiredness as Requiredness from flyde.node import Component as Component from typing import Any @@ -52,3 +52,10 @@ class GetAttribute(Component): outputs: Incomplete def __init__(self, **kwargs) -> None: ... def process(self, object: Any, key: str): ... + +class Http(Component): + """Http component makes HTTP requests with urllib.""" + inputs: Incomplete + outputs: Incomplete + def __init__(self, **kwargs) -> None: ... + def process(self, url: str, method: str, headers: dict | None = None, params: dict | None = None, data: dict | None = None): ... diff --git a/tests/test_stdlib.py b/tests/test_stdlib.py index 43c1e4e..30902b1 100644 --- a/tests/test_stdlib.py +++ b/tests/test_stdlib.py @@ -1,9 +1,12 @@ import unittest from queue import Queue from types import SimpleNamespace +from unittest.mock import patch, MagicMock +from urllib import error +from http import client from flyde.io import EOF, InputConfig, InputType -from flyde.stdlib import Conditional, GetAttribute, InlineValue, _ConditionConfig, _ConditionType +from flyde.stdlib import Conditional, GetAttribute, Http, InlineValue, _ConditionConfig, _ConditionType class TestInlineValue(unittest.TestCase): @@ -398,3 +401,234 @@ def test_get_attribute(self): node.stopped.wait() self.assertTrue(node.stopped.is_set()) + + +class TestHttp(unittest.TestCase): + @patch('urllib.request.urlopen') + def test_http_get(self, mock_urlopen): + mock_response = MagicMock() + mock_response.status = 200 + mock_response.headers = {'Content-Type': 'application/json'} + mock_response.read.return_value = b'{"message": "Hello, World!"}' + mock_response.__enter__.return_value = mock_response + mock_urlopen.return_value = mock_response + + test_case = { + "name": "simple GET request", + "config": { + "method": InputConfig(type=InputType.STRING, value="GET"), + "url": InputConfig(type=InputType.STRING, value="https://example.com/api"), + "headers": InputConfig(type=InputType.JSON, value={"User-Agent": "PyFlyde Test"}), + "params": InputConfig(type=InputType.JSON, value={"q": "test"}), + }, + } + + data_q = Queue() + + node = Http(id="test_http", config=test_case["config"]) + node.outputs["data"].connect(data_q) + + node.process( + url="https://example.com/api", + method="GET", + headers={"User-Agent": "PyFlyde Test"}, + params={"q": "test"} + ) + + self.assertEqual({"message": "Hello, World!"}, data_q.get_nowait()) + + # Verify the mock was called with the expected arguments + mock_urlopen.assert_called_once() + args, _ = mock_urlopen.call_args + self.assertTrue("https://example.com/api?q=test" in str(args[0].full_url)) + self.assertEqual("GET", args[0].method) + + # Check that the User-Agent header was set + user_agent = None + for key, value in args[0].headers.items(): + if key.lower() == "user-agent": + user_agent = value + break + self.assertEqual("PyFlyde Test", user_agent) + + @patch('urllib.request.urlopen') + def test_http_html_response(self, mock_urlopen): + mock_response = MagicMock() + mock_response.status = 200 + mock_response.headers = {'Content-Type': 'text/html; charset=utf-8'} + mock_response.read.return_value = b'

Test Page

' + mock_response.__enter__.return_value = mock_response + mock_urlopen.return_value = mock_response + + data_q = Queue() + + node = Http(id="test_http", config={ + "method": InputConfig(type=InputType.STRING, value="GET"), + "url": InputConfig(type=InputType.STRING, value="https://example.com"), + }) + node.outputs["data"].connect(data_q) + + node.process( + url="https://example.com", + method="GET" + ) + + self.assertEqual("

Test Page

", data_q.get_nowait()) + + mock_urlopen.assert_called_once() + args, _ = mock_urlopen.call_args + self.assertEqual("https://example.com", str(args[0].full_url)) + self.assertEqual("GET", args[0].method) + + @patch('urllib.request.urlopen') + def test_http_binary_response(self, mock_urlopen): + mock_response = MagicMock() + mock_response.status = 200 + mock_response.headers = {'Content-Type': 'application/octet-stream'} + binary_data = b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00' # Start of a PNG file + mock_response.read.return_value = binary_data + mock_response.__enter__.return_value = mock_response + mock_urlopen.return_value = mock_response + + data_q = Queue() + + node = Http(id="test_http", config={ + "method": InputConfig(type=InputType.STRING, value="GET"), + "url": InputConfig(type=InputType.STRING, value="https://example.com/image.png"), + }) + node.outputs["data"].connect(data_q) + + node.process( + url="https://example.com/image.png", + method="GET" + ) + + # Binary data should be returned as is + self.assertEqual(binary_data, data_q.get_nowait()) + + mock_urlopen.assert_called_once() + args, _ = mock_urlopen.call_args + self.assertEqual("https://example.com/image.png", str(args[0].full_url)) + self.assertEqual("GET", args[0].method) + + @patch('urllib.request.urlopen') + def test_http_non_utf8_encoding(self, mock_urlopen): + mock_response = MagicMock() + mock_response.status = 200 + mock_response.headers = {'Content-Type': 'text/html; charset=ISO-8859-1'} + # Latin-1 encoded text with special characters + latin1_data = b'Espa\xf1ol Fran\xe7ais Portugu\xeas' + mock_response.read.return_value = latin1_data + mock_response.__enter__.return_value = mock_response + mock_urlopen.return_value = mock_response + + data_q = Queue() + + node = Http(id="test_http", config={ + "method": InputConfig(type=InputType.STRING, value="GET"), + "url": InputConfig(type=InputType.STRING, value="https://example.com/latin1.html"), + }) + node.outputs["data"].connect(data_q) + + node.process( + url="https://example.com/latin1.html", + method="GET" + ) + + # Should be properly decoded using ISO-8859-1 charset + self.assertEqual("Español Français Português", data_q.get_nowait()) + + mock_urlopen.assert_called_once() + args, _ = mock_urlopen.call_args + self.assertEqual("https://example.com/latin1.html", str(args[0].full_url)) + self.assertEqual("GET", args[0].method) + + @patch('urllib.request.urlopen') + def test_http_post(self, mock_urlopen): + mock_response = MagicMock() + mock_response.status = 201 + mock_response.headers = {'Content-Type': 'application/json'} + mock_response.read.return_value = b'{"id": 1, "success": true}' + mock_response.__enter__.return_value = mock_response + mock_urlopen.return_value = mock_response + + test_case = { + "name": "POST request with data", + "config": { + "method": InputConfig(type=InputType.STRING, value="POST"), + "url": InputConfig(type=InputType.STRING, value="https://example.com/api/users"), + "headers": InputConfig(type=InputType.JSON, value={"User-Agent": "PyFlyde Test"}), + "data": InputConfig(type=InputType.JSON, value={"name": "Test User", "email": "test@example.com"}), + }, + } + + data_q = Queue() + + node = Http(id="test_http", config=test_case["config"]) + node.outputs["data"].connect(data_q) + + node.process( + url="https://example.com/api/users", + method="POST", + headers={"User-Agent": "PyFlyde Test"}, + data={"name": "Test User", "email": "test@example.com"} + ) + + self.assertEqual({"id": 1, "success": True}, data_q.get_nowait()) + + # Verify the mock was called with the expected arguments + mock_urlopen.assert_called_once() + args, kwargs = mock_urlopen.call_args + self.assertEqual("https://example.com/api/users", str(args[0].full_url)) + self.assertEqual("POST", args[0].method) + + # Check that the User-Agent header was set + user_agent = None + for key, value in args[0].headers.items(): + if key.lower() == "user-agent": + user_agent = value + break + self.assertEqual("PyFlyde Test", user_agent) + + self.assertEqual(b'{"name": "Test User", "email": "test@example.com"}', kwargs["data"]) + + @patch('urllib.request.urlopen') + def test_http_error(self, mock_urlopen): + # Create a mock headers object for HTTPError + headers = client.HTTPMessage() + headers.add_header('Content-Type', 'text/plain') + + mock_urlopen.side_effect = error.HTTPError( + url="https://example.com/api/error", + code=404, + msg="Not Found", + hdrs=headers, + fp=None, + ) + + test_case = { + "name": "HTTP error handling", + "config": { + "method": InputConfig(type=InputType.STRING, value="GET"), + "url": InputConfig(type=InputType.STRING, value="https://example.com/api/error"), + }, + } + + data_q = Queue() + + node = Http(id="test_http", config=test_case["config"]) + node.outputs["data"].connect(data_q) + + with self.assertRaises(error.HTTPError) as context: + node.process( + url="https://example.com/api/error", + method="GET" + ) + + self.assertEqual(404, context.exception.code) + self.assertEqual("Not Found", context.exception.msg) + + # Verify the mock was called + mock_urlopen.assert_called_once() + args, _ = mock_urlopen.call_args + self.assertEqual("https://example.com/api/error", str(args[0].full_url)) From ff9c2c9a54f2b5941bfcbd2cc812278d4f2853c6 Mon Sep 17 00:00:00 2001 From: Vladimir Sibirov Date: Mon, 7 Jul 2025 21:21:00 +0200 Subject: [PATCH 07/18] Rename stdlib to nodes --- examples/Clustering.flyde | 8 +- flyde/{stdlib.py => nodes.py} | 0 flyde/{stdlib.pyi => nodes.pyi} | 0 tests/Repeat3Times.flyde | 2 +- tests/TestFanIn.flyde | 2 +- tests/TestFanInGraph.flyde | 2 +- tests/TestInOutFlow.flyde | 4 +- tests/TestIsolatedFlow.flyde | 2 +- tests/TestNestedFlow.flyde | 43 ++++---- tests/TestStdlib.flyde | 51 +++++++++ tests/test_stdlib.py | 178 ++++++++++++++++++++------------ 11 files changed, 197 insertions(+), 95 deletions(-) rename flyde/{stdlib.py => nodes.py} (100%) rename flyde/{stdlib.pyi => nodes.pyi} (100%) create mode 100644 tests/TestStdlib.flyde diff --git a/examples/Clustering.flyde b/examples/Clustering.flyde index 3450f11..ec4b952 100644 --- a/examples/Clustering.flyde +++ b/examples/Clustering.flyde @@ -34,7 +34,7 @@ node: type: code source: type: package - data: "@flyde/stdlib" + data: "@flyde/nodes" - pos: x: -436.9881726074219 y: 60.95093536376953 @@ -102,7 +102,7 @@ node: type: code source: type: package - data: "@flyde/stdlib" + data: "@flyde/nodes" - pos: x: 234.12208740234377 y: 410.2254124450684 @@ -166,7 +166,7 @@ node: type: code source: type: package - data: "@flyde/stdlib" + data: "@flyde/nodes" - pos: x: 234.12473022460927 y: 310.8023236303703 @@ -186,7 +186,7 @@ node: type: code source: type: package - data: "@flyde/stdlib" + data: "@flyde/nodes" connections: - from: insId: mc4t4fqqezd1gns4fb7ckxmc diff --git a/flyde/stdlib.py b/flyde/nodes.py similarity index 100% rename from flyde/stdlib.py rename to flyde/nodes.py diff --git a/flyde/stdlib.pyi b/flyde/nodes.pyi similarity index 100% rename from flyde/stdlib.pyi rename to flyde/nodes.pyi diff --git a/tests/Repeat3Times.flyde b/tests/Repeat3Times.flyde index 4adf691..81ab229 100644 --- a/tests/Repeat3Times.flyde +++ b/tests/Repeat3Times.flyde @@ -33,7 +33,7 @@ node: type: code source: type: package - data: "@flyde/stdlib" + data: "@flyde/nodes" - pos: x: 109.24334716796875 y: 55.590057373046875 diff --git a/tests/TestFanIn.flyde b/tests/TestFanIn.flyde index 31397a8..35dd6d6 100644 --- a/tests/TestFanIn.flyde +++ b/tests/TestFanIn.flyde @@ -62,7 +62,7 @@ node: type: code source: type: package - data: "@flyde/stdlib" + data: "@flyde/nodes" connections: - from: insId: Format-4s04bag diff --git a/tests/TestFanInGraph.flyde b/tests/TestFanInGraph.flyde index 4b191a7..b9cc7be 100644 --- a/tests/TestFanInGraph.flyde +++ b/tests/TestFanInGraph.flyde @@ -1,5 +1,5 @@ imports: - "@flyde/stdlib": + "@flyde/nodes": - InlineValue components.flyde.ts: - Format diff --git a/tests/TestInOutFlow.flyde b/tests/TestInOutFlow.flyde index edb9d63..3b223e3 100644 --- a/tests/TestInOutFlow.flyde +++ b/tests/TestInOutFlow.flyde @@ -33,7 +33,7 @@ node: type: code source: type: package - data: "@flyde/stdlib" + data: "@flyde/nodes" - pos: x: 241.42891601562496 y: 13.463867167501462 @@ -70,7 +70,7 @@ node: type: code source: type: package - data: "@flyde/stdlib" + data: "@flyde/nodes" connections: - from: insId: Echo-h3049mb diff --git a/tests/TestIsolatedFlow.flyde b/tests/TestIsolatedFlow.flyde index 12777e1..0789c69 100644 --- a/tests/TestIsolatedFlow.flyde +++ b/tests/TestIsolatedFlow.flyde @@ -14,7 +14,7 @@ node: type: code source: type: package - data: "@flyde/stdlib" + data: "@flyde/nodes" - pos: x: -83.18318359374999 y: 60.36263549804687 diff --git a/tests/TestNestedFlow.flyde b/tests/TestNestedFlow.flyde index 56e3701..4e55af3 100644 --- a/tests/TestNestedFlow.flyde +++ b/tests/TestNestedFlow.flyde @@ -1,10 +1,4 @@ -imports: - "@flyde/stdlib": [] - Repeat3Times.flyde: - - Repeat3Times - components.flyde.ts: - - Echo - - RepeatWordNTimes +imports: {} node: instances: - pos: @@ -13,12 +7,24 @@ node: id: Repeat3Times-u6049uf inputConfig: {} nodeId: Repeat3Times + type: visual + source: + type: file + data: Repeat3Times.flyde - pos: x: -196.171416015625 y: 107.68781005859375 id: Echo-co1498h inputConfig: {} nodeId: Echo + config: + inp: + type: dynamic + value: "{{inp}}" + type: code + source: + type: file + data: components.flyde.ts - pos: x: -195.52671508789064 y: 269.8743734741211 @@ -27,19 +33,18 @@ node: times: mode: sticky nodeId: RepeatWordNTimes + config: + word: + type: dynamic + value: "{{word}}" + times: + type: dynamic + value: "{{times}}" + type: code + source: + type: file + data: components.flyde.ts connections: - - from: - insId: __this - pinId: inp - to: - insId: Repeat3Times-u6049uf - pinId: word - - from: - insId: Repeat3Times-u6049uf - pinId: word3x - to: - insId: Echo-co1498h - pinId: inp - from: insId: Echo-co1498h pinId: out diff --git a/tests/TestStdlib.flyde b/tests/TestStdlib.flyde new file mode 100644 index 0000000..1b71f14 --- /dev/null +++ b/tests/TestStdlib.flyde @@ -0,0 +1,51 @@ +imports: {} +node: + instances: + - pos: + x: -150 + y: 20 + id: Http-dd04gea + inputConfig: {} + nodeId: Http + config: + method: + type: select + value: GET + url: + type: string + value: https://www.flyde.dev + data: + type: json + value: {} + params: + type: json + value: + ref: pyflyde + headers: + type: json + value: + User-Agent: PyFlyde v0.1 + type: code + source: + type: package + data: "@flyde/nodes" + connections: + - from: + insId: Http-dd04gea + pinId: data + to: + insId: __this + pinId: response + id: Example + inputs: {} + outputs: + response: + delayed: false + inputsPosition: {} + outputsPosition: + result: + x: -23.264428942324532 + y: 237.25953921502617 + response: + x: 124.5 + y: 19 diff --git a/tests/test_stdlib.py b/tests/test_stdlib.py index 30902b1..492d549 100644 --- a/tests/test_stdlib.py +++ b/tests/test_stdlib.py @@ -6,7 +6,14 @@ from http import client from flyde.io import EOF, InputConfig, InputType -from flyde.stdlib import Conditional, GetAttribute, Http, InlineValue, _ConditionConfig, _ConditionType +from flyde.nodes import ( + Conditional, + GetAttribute, + Http, + InlineValue, + _ConditionConfig, + _ConditionType, +) class TestInlineValue(unittest.TestCase): @@ -16,7 +23,10 @@ def test_inline_value(self): "outputs": {"value": "Hello"}, } out_q = Queue() - node = InlineValue(id="test_inline_value", config={"value": InputConfig(type=InputType.STRING, value="Hello")}) + node = InlineValue( + id="test_inline_value", + config={"value": InputConfig(type=InputType.STRING, value="Hello")}, + ) node.outputs["value"].connect(out_q) node.run() self.assertEqual(test_case["outputs"]["value"], out_q.get()) @@ -29,7 +39,10 @@ def test_inline_value_dict(self): "outputs": {"value": "Hello"}, } out_q = Queue() - node = InlineValue(id="test_inline_value", config={"value": InputConfig(type=InputType.STRING, value="Hello")}) + node = InlineValue( + id="test_inline_value", + config={"value": InputConfig(type=InputType.STRING, value="Hello")}, + ) node.outputs["value"].connect(out_q) node.run() self.assertEqual(test_case["outputs"]["value"], out_q.get()) @@ -168,7 +181,9 @@ def test_conditional(self): "leftOperand": InputConfig( type=InputType.DYNAMIC, ), - "rightOperand": InputConfig(type=InputType.STRING, value="this is not important"), + "rightOperand": InputConfig( + type=InputType.STRING, value="this is not important" + ), "condition": _ConditionConfig( type=_ConditionType.Exists, ), @@ -188,7 +203,9 @@ def test_conditional(self): "leftOperand": InputConfig( type=InputType.DYNAMIC, ), - "rightOperand": InputConfig(type=InputType.STRING, value="this is not important"), + "rightOperand": InputConfig( + type=InputType.STRING, value="this is not important" + ), "condition": _ConditionConfig( type=_ConditionType.NotExists, ), @@ -211,7 +228,9 @@ def test_conditional(self): "rightOperand": InputConfig( type=InputType.DYNAMIC, ), - "condition": {"type": "UNSUPPORTED"}, # Will cause a ValueError in parse_config + "condition": { + "type": "UNSUPPORTED" + }, # Will cause a ValueError in parse_config }, "inputs": { "leftOperand": ["Apple", "Banana", "apple", EOF], @@ -231,7 +250,9 @@ def test_conditional(self): if "raises" in test_case and test_case["raises"] is not None: with self.assertRaises(test_case["raises"]): - node = Conditional(id="test_conditional", config=test_case["config"]) + node = Conditional( + id="test_conditional", config=test_case["config"] + ) continue node = Conditional(id="test_conditional", config=test_case["config"]) @@ -395,7 +416,9 @@ def test_get_attribute(self): node.run() for i in range(len(test_case["inputs"]["object"])): obj_q.put(test_case["inputs"]["object"][i]) - if len(test_case["inputs"]["key"]) > 0 and i < len(test_case["inputs"]["key"]): + if len(test_case["inputs"]["key"]) > 0 and i < len( + test_case["inputs"]["key"] + ): attr_q.put(test_case["inputs"]["key"][i]) self.assertEqual(test_case["outputs"][i], out_q.get()) @@ -404,11 +427,11 @@ def test_get_attribute(self): class TestHttp(unittest.TestCase): - @patch('urllib.request.urlopen') + @patch("urllib.request.urlopen") def test_http_get(self, mock_urlopen): mock_response = MagicMock() mock_response.status = 200 - mock_response.headers = {'Content-Type': 'application/json'} + mock_response.headers = {"Content-Type": "application/json"} mock_response.read.return_value = b'{"message": "Hello, World!"}' mock_response.__enter__.return_value = mock_response mock_urlopen.return_value = mock_response @@ -417,8 +440,12 @@ def test_http_get(self, mock_urlopen): "name": "simple GET request", "config": { "method": InputConfig(type=InputType.STRING, value="GET"), - "url": InputConfig(type=InputType.STRING, value="https://example.com/api"), - "headers": InputConfig(type=InputType.JSON, value={"User-Agent": "PyFlyde Test"}), + "url": InputConfig( + type=InputType.STRING, value="https://example.com/api" + ), + "headers": InputConfig( + type=InputType.JSON, value={"User-Agent": "PyFlyde Test"} + ), "params": InputConfig(type=InputType.JSON, value={"q": "test"}), }, } @@ -432,7 +459,7 @@ def test_http_get(self, mock_urlopen): url="https://example.com/api", method="GET", headers={"User-Agent": "PyFlyde Test"}, - params={"q": "test"} + params={"q": "test"}, ) self.assertEqual({"message": "Hello, World!"}, data_q.get_nowait()) @@ -442,7 +469,7 @@ def test_http_get(self, mock_urlopen): args, _ = mock_urlopen.call_args self.assertTrue("https://example.com/api?q=test" in str(args[0].full_url)) self.assertEqual("GET", args[0].method) - + # Check that the User-Agent header was set user_agent = None for key, value in args[0].headers.items(): @@ -451,57 +478,66 @@ def test_http_get(self, mock_urlopen): break self.assertEqual("PyFlyde Test", user_agent) - @patch('urllib.request.urlopen') + @patch("urllib.request.urlopen") def test_http_html_response(self, mock_urlopen): mock_response = MagicMock() mock_response.status = 200 - mock_response.headers = {'Content-Type': 'text/html; charset=utf-8'} - mock_response.read.return_value = b'

Test Page

' + mock_response.headers = {"Content-Type": "text/html; charset=utf-8"} + mock_response.read.return_value = ( + b"

Test Page

" + ) mock_response.__enter__.return_value = mock_response mock_urlopen.return_value = mock_response data_q = Queue() - node = Http(id="test_http", config={ - "method": InputConfig(type=InputType.STRING, value="GET"), - "url": InputConfig(type=InputType.STRING, value="https://example.com"), - }) + node = Http( + id="test_http", + config={ + "method": InputConfig(type=InputType.STRING, value="GET"), + "url": InputConfig(type=InputType.STRING, value="https://example.com"), + }, + ) node.outputs["data"].connect(data_q) - node.process( - url="https://example.com", - method="GET" - ) + node.process(url="https://example.com", method="GET") - self.assertEqual("

Test Page

", data_q.get_nowait()) + self.assertEqual( + "

Test Page

", + data_q.get_nowait(), + ) mock_urlopen.assert_called_once() args, _ = mock_urlopen.call_args self.assertEqual("https://example.com", str(args[0].full_url)) self.assertEqual("GET", args[0].method) - @patch('urllib.request.urlopen') + @patch("urllib.request.urlopen") def test_http_binary_response(self, mock_urlopen): mock_response = MagicMock() mock_response.status = 200 - mock_response.headers = {'Content-Type': 'application/octet-stream'} - binary_data = b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00' # Start of a PNG file + mock_response.headers = {"Content-Type": "application/octet-stream"} + binary_data = ( + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00" # Start of a PNG file + ) mock_response.read.return_value = binary_data mock_response.__enter__.return_value = mock_response mock_urlopen.return_value = mock_response data_q = Queue() - node = Http(id="test_http", config={ - "method": InputConfig(type=InputType.STRING, value="GET"), - "url": InputConfig(type=InputType.STRING, value="https://example.com/image.png"), - }) + node = Http( + id="test_http", + config={ + "method": InputConfig(type=InputType.STRING, value="GET"), + "url": InputConfig( + type=InputType.STRING, value="https://example.com/image.png" + ), + }, + ) node.outputs["data"].connect(data_q) - node.process( - url="https://example.com/image.png", - method="GET" - ) + node.process(url="https://example.com/image.png", method="GET") # Binary data should be returned as is self.assertEqual(binary_data, data_q.get_nowait()) @@ -511,29 +547,31 @@ def test_http_binary_response(self, mock_urlopen): self.assertEqual("https://example.com/image.png", str(args[0].full_url)) self.assertEqual("GET", args[0].method) - @patch('urllib.request.urlopen') + @patch("urllib.request.urlopen") def test_http_non_utf8_encoding(self, mock_urlopen): mock_response = MagicMock() mock_response.status = 200 - mock_response.headers = {'Content-Type': 'text/html; charset=ISO-8859-1'} + mock_response.headers = {"Content-Type": "text/html; charset=ISO-8859-1"} # Latin-1 encoded text with special characters - latin1_data = b'Espa\xf1ol Fran\xe7ais Portugu\xeas' + latin1_data = b"Espa\xf1ol Fran\xe7ais Portugu\xeas" mock_response.read.return_value = latin1_data mock_response.__enter__.return_value = mock_response mock_urlopen.return_value = mock_response data_q = Queue() - node = Http(id="test_http", config={ - "method": InputConfig(type=InputType.STRING, value="GET"), - "url": InputConfig(type=InputType.STRING, value="https://example.com/latin1.html"), - }) + node = Http( + id="test_http", + config={ + "method": InputConfig(type=InputType.STRING, value="GET"), + "url": InputConfig( + type=InputType.STRING, value="https://example.com/latin1.html" + ), + }, + ) node.outputs["data"].connect(data_q) - node.process( - url="https://example.com/latin1.html", - method="GET" - ) + node.process(url="https://example.com/latin1.html", method="GET") # Should be properly decoded using ISO-8859-1 charset self.assertEqual("Español Français Português", data_q.get_nowait()) @@ -543,11 +581,11 @@ def test_http_non_utf8_encoding(self, mock_urlopen): self.assertEqual("https://example.com/latin1.html", str(args[0].full_url)) self.assertEqual("GET", args[0].method) - @patch('urllib.request.urlopen') + @patch("urllib.request.urlopen") def test_http_post(self, mock_urlopen): mock_response = MagicMock() mock_response.status = 201 - mock_response.headers = {'Content-Type': 'application/json'} + mock_response.headers = {"Content-Type": "application/json"} mock_response.read.return_value = b'{"id": 1, "success": true}' mock_response.__enter__.return_value = mock_response mock_urlopen.return_value = mock_response @@ -556,9 +594,16 @@ def test_http_post(self, mock_urlopen): "name": "POST request with data", "config": { "method": InputConfig(type=InputType.STRING, value="POST"), - "url": InputConfig(type=InputType.STRING, value="https://example.com/api/users"), - "headers": InputConfig(type=InputType.JSON, value={"User-Agent": "PyFlyde Test"}), - "data": InputConfig(type=InputType.JSON, value={"name": "Test User", "email": "test@example.com"}), + "url": InputConfig( + type=InputType.STRING, value="https://example.com/api/users" + ), + "headers": InputConfig( + type=InputType.JSON, value={"User-Agent": "PyFlyde Test"} + ), + "data": InputConfig( + type=InputType.JSON, + value={"name": "Test User", "email": "test@example.com"}, + ), }, } @@ -571,7 +616,7 @@ def test_http_post(self, mock_urlopen): url="https://example.com/api/users", method="POST", headers={"User-Agent": "PyFlyde Test"}, - data={"name": "Test User", "email": "test@example.com"} + data={"name": "Test User", "email": "test@example.com"}, ) self.assertEqual({"id": 1, "success": True}, data_q.get_nowait()) @@ -581,7 +626,7 @@ def test_http_post(self, mock_urlopen): args, kwargs = mock_urlopen.call_args self.assertEqual("https://example.com/api/users", str(args[0].full_url)) self.assertEqual("POST", args[0].method) - + # Check that the User-Agent header was set user_agent = None for key, value in args[0].headers.items(): @@ -589,15 +634,17 @@ def test_http_post(self, mock_urlopen): user_agent = value break self.assertEqual("PyFlyde Test", user_agent) - - self.assertEqual(b'{"name": "Test User", "email": "test@example.com"}', kwargs["data"]) - @patch('urllib.request.urlopen') + self.assertEqual( + b'{"name": "Test User", "email": "test@example.com"}', kwargs["data"] + ) + + @patch("urllib.request.urlopen") def test_http_error(self, mock_urlopen): # Create a mock headers object for HTTPError headers = client.HTTPMessage() - headers.add_header('Content-Type', 'text/plain') - + headers.add_header("Content-Type", "text/plain") + mock_urlopen.side_effect = error.HTTPError( url="https://example.com/api/error", code=404, @@ -610,7 +657,9 @@ def test_http_error(self, mock_urlopen): "name": "HTTP error handling", "config": { "method": InputConfig(type=InputType.STRING, value="GET"), - "url": InputConfig(type=InputType.STRING, value="https://example.com/api/error"), + "url": InputConfig( + type=InputType.STRING, value="https://example.com/api/error" + ), }, } @@ -620,11 +669,8 @@ def test_http_error(self, mock_urlopen): node.outputs["data"].connect(data_q) with self.assertRaises(error.HTTPError) as context: - node.process( - url="https://example.com/api/error", - method="GET" - ) - + node.process(url="https://example.com/api/error", method="GET") + self.assertEqual(404, context.exception.code) self.assertEqual("Not Found", context.exception.msg) From 85a6971c306b6accc64c53edc6177d8b42a9d35d Mon Sep 17 00:00:00 2001 From: Vladimir Sibirov Date: Sat, 12 Jul 2025 20:04:15 +0200 Subject: [PATCH 08/18] Fix nested flows --- flyde/flow.py | 8 ++- flyde/io.py | 11 +-- tests/Repeat3Times.flyde | 4 +- tests/TestFanInGraph.flyde | 59 ++++++++++------ tests/TestNestedFlow.flyde | 48 ++++++++----- tests/components.py | 4 +- tests/test_flow.py | 139 ++++++++++++++++++------------------- 7 files changed, 153 insertions(+), 120 deletions(-) diff --git a/flyde/flow.py b/flyde/flow.py index ea412d6..e097e4b 100644 --- a/flyde/flow.py +++ b/flyde/flow.py @@ -26,7 +26,7 @@ def __init__(self, imports: dict[str, list[str]]): def _preload_imports(self, base_path: str, imports: dict[str, list[str]]): if not imports: return - + for module, classes in imports.items(): logger.debug(f"Importing {module}") # If module name ends with .flyde it's a Graph @@ -51,7 +51,8 @@ def _preload_imports(self, base_path: str, imports: dict[str, list[str]]): def _load_graph(self, name: str, path: str): """Loads a graph YAML.""" - yml = load_yaml_file(path) + full_path = os.path.join(self._base_path, path) + yml = load_yaml_file(full_path) if not isinstance(yml, dict): raise ValueError(f"Invalid YAML file {path}") # Save the blueprint YAML for the graph to be instantiated later @@ -97,7 +98,8 @@ def create_component(self, name: str, args: InstanceArgs): def factory(self, class_name: str, args: InstanceArgs): """Factory method to create a node from a class name and arguments. - It is used by the runtime to create nodes from the YAML definition or on the fly.""" + It is used by the runtime to create nodes from the YAML definition or on the fly. + """ if args.type == InstanceType.VISUAL: return self.create_graph(class_name, args) diff --git a/flyde/io.py b/flyde/io.py index c60f61b..1a53792 100644 --- a/flyde/io.py +++ b/flyde/io.py @@ -175,10 +175,13 @@ def ref_count(self) -> int: def apply_config(self, config: InputConfig): """Apply config from the flyde flow to the input.""" self._value = config.value - if config.type == InputType.DYNAMIC: - self._input_mode = InputMode.QUEUE - else: - self._input_mode = InputMode.STICKY + # If input mode is STICKY already, stick to it + # as it may be important for the node to function correctly + if self._input_mode != InputMode.STICKY: + if config.type == InputType.DYNAMIC: + self._input_mode = InputMode.QUEUE + else: + self._input_mode = InputMode.STICKY # Apply Python type hint based on supported config type if config.type != InputType.DYNAMIC and self.type is None: diff --git a/tests/Repeat3Times.flyde b/tests/Repeat3Times.flyde index 81ab229..6977be4 100644 --- a/tests/Repeat3Times.flyde +++ b/tests/Repeat3Times.flyde @@ -3,7 +3,7 @@ node: instances: - pos: x: -105.91999999999999 - y: 47.08 + y: 49.08 id: RepeatWordNTimes-h30493h inputConfig: times: @@ -91,4 +91,4 @@ node: word3x: x: 345.94718750000004 y: 57.28312866210939 - description: For each input string, sends a string with the same conent repeated 3 times + description: For each input string, sends a string with the same content repeated 3 times diff --git a/tests/TestFanInGraph.flyde b/tests/TestFanInGraph.flyde index b9cc7be..c048c85 100644 --- a/tests/TestFanInGraph.flyde +++ b/tests/TestFanInGraph.flyde @@ -1,11 +1,4 @@ -imports: - "@flyde/nodes": - - InlineValue - components.flyde.ts: - - Format - - Capitalize - Repeat3Times.flyde: - - Repeat3Times +imports: {} node: instances: - pos: @@ -14,32 +7,58 @@ node: id: Format-4s04bag inputConfig: {} nodeId: Format + config: + inp: + type: dynamic + value: "{{inp}}" + format: + type: dynamic + value: "{{format}}" + type: code + source: + type: file + data: components.flyde.ts - pos: - x: -44.16827392578125 + x: -58.16827392578125 y: 63.02734375 id: Capitalize-ch04buf inputConfig: {} nodeId: Capitalize + config: + inp: + type: dynamic + value: "{{inp}}" + type: code + source: + type: file + data: components.flyde.ts - pos: - x: -328.1567211914063 - y: -90.1646710205078 + x: -601.1567211914063 + y: 121.8353289794922 id: xzi4aah4ewf1iw7q4a3jz9oj inputConfig: {} - nodeId: InlineValue__xzi4aah4ewf1iw7q4a3jz9oj - macroId: InlineValue - macroData: + nodeId: InlineValue + config: type: type: string value: string value: type: string value: Hello, {inp}! + type: code + source: + type: package + data: "@flyde/nodes" - pos: - x: -185.60189453125 - y: 240.50311500933114 + x: 187.39810546875 + y: 119.50311500933117 id: Repeat3Times-ou04byd inputConfig: {} nodeId: Repeat3Times + type: visual + source: + type: file + data: Repeat3Times.flyde connections: - from: insId: __this @@ -92,12 +111,12 @@ node: delayed: false inputsPosition: str: - x: -119.99874877929688 - y: -99.46359436035156 + x: -593.9987487792969 + y: -21.463594360351564 outputsPosition: result: x: -23.264428942324532 y: 237.25953921502617 out: - x: -120.79533203125001 - y: 390.6666717529297 + x: 396.20466796874996 + y: 119.66667175292969 diff --git a/tests/TestNestedFlow.flyde b/tests/TestNestedFlow.flyde index 4e55af3..5eb664e 100644 --- a/tests/TestNestedFlow.flyde +++ b/tests/TestNestedFlow.flyde @@ -2,18 +2,8 @@ imports: {} node: instances: - pos: - x: -234.30030517578126 - y: -38.32338623046875 - id: Repeat3Times-u6049uf - inputConfig: {} - nodeId: Repeat3Times - type: visual - source: - type: file - data: Repeat3Times.flyde - - pos: - x: -196.171416015625 - y: 107.68781005859375 + x: -465.54763671875 + y: 195.6628924560547 id: Echo-co1498h inputConfig: {} nodeId: Echo @@ -44,6 +34,16 @@ node: source: type: file data: components.flyde.ts + - pos: + x: -674.5008868408204 + y: 195.91538436889653 + id: Repeat3Times-dfk3brg0 + inputConfig: {} + nodeId: Repeat3Times + type: visual + source: + type: file + data: Repeat3Times.flyde connections: - from: insId: Echo-co1498h @@ -63,6 +63,18 @@ node: to: insId: RepeatWordNTimes-pp249oy pinId: times + - from: + insId: Repeat3Times-dfk3brg0 + pinId: word3x + to: + insId: Echo-co1498h + pinId: inp + - from: + insId: __this + pinId: inp + to: + insId: Repeat3Times-dfk3brg0 + pinId: word id: TestNestedFlow inputs: inp: @@ -74,19 +86,19 @@ node: delayed: false inputsPosition: inp: - x: -172.50668701171875 - y: -161.04999633789063 + x: -854.638095703125 + y: 195.21616271972655 x: x: 37.526656494140624 y: -159.08574951171875 n: - x: 37.526656494140624 - y: -159.08574951171875 + x: -448.31001342773436 + y: 288.3494387817383 outputsPosition: result: x: -23.264428942324532 y: 237.25953921502617 out: - x: -110.85763549804688 - y: 444.78157318115234 + x: 20.572113037109375 + y: 280.7653988647461 description: Repeats input 3xN times diff --git a/tests/components.py b/tests/components.py index 2e91cf7..4609042 100644 --- a/tests/components.py +++ b/tests/components.py @@ -7,9 +7,7 @@ class Format(Component): inputs = { "inp": Input(description="The input"), - "format": Input( - description="The format string", type=str, mode=InputMode.STICKY - ), + "format": Input(description="The format string", type=str, mode=InputMode.STICKY), } outputs = { "out": Output(description="The formatted output", type=str), diff --git a/tests/test_flow.py b/tests/test_flow.py index 3cb887e..0cddcca 100644 --- a/tests/test_flow.py +++ b/tests/test_flow.py @@ -35,38 +35,38 @@ def test_flow(self): self.assertTrue(flow.stopped.is_set()) -# class TestNestedFlow(unittest.TestCase): -# def test_flow(self): -# test_case = { -# "inputs": { -# "inp": ["Hello", "World", "!", EOF], -# "n": [1, 2, EOF], -# }, -# "outputs": [ -# "HELLOHELLOHELLO", -# "WORLDWORLDWORLDWORLDWORLDWORLD", -# "!!!!!!", -# EOF, -# ], -# } -# flow = Flow.from_file("tests/TestNestedFlow.flyde") - -# inp_q = flow.node.inputs["inp"].queue -# n_q = flow.node.inputs["n"].queue -# out_q = Queue() -# flow.node.outputs["out"].connect(out_q) - -# flow.run() - -# for i, inp in enumerate(test_case["inputs"]["inp"]): -# inp_q.put(inp) -# if i < len(test_case["inputs"]["n"]): -# n_q.put(test_case["inputs"]["n"][i]) -# out = out_q.get() -# self.assertEqual(test_case["outputs"][i], out) - -# flow.stopped.wait() -# self.assertTrue(flow.stopped.is_set()) +class TestNestedFlow(unittest.TestCase): + def test_flow(self): + test_case = { + "inputs": { + "inp": ["Hello", "World", "!", EOF], + "n": [1, 2, EOF], + }, + "outputs": [ + "HELLOHELLOHELLO", + "WORLDWORLDWORLDWORLDWORLDWORLD", + "!!!!!!", + EOF, + ], + } + flow = Flow.from_file("tests/TestNestedFlow.flyde") + + inp_q = flow.node.inputs["inp"].queue + n_q = flow.node.inputs["n"].queue + out_q = Queue() + flow.node.outputs["out"].connect(out_q) + + flow.run() + + for i, inp in enumerate(test_case["inputs"]["inp"]): + inp_q.put(inp) + if i < len(test_case["inputs"]["n"]): + n_q.put(test_case["inputs"]["n"][i]) + out = out_q.get() + self.assertEqual(test_case["outputs"][i], out) + + flow.stopped.wait() + self.assertTrue(flow.stopped.is_set()) class TestFanInFlow(unittest.TestCase): @@ -104,42 +104,41 @@ def test_with_component(self): flow.stopped.wait() self.assertTrue(flow.stopped.is_set()) + def test_with_graph(self): + test_case = { + "inputs": ["John", EOF], + "outputs": [ + "JOHNJOHNJOHN", + "JOHNJOHNJOHN", + "HELLO, JOHN!HELLO, JOHN!HELLO, JOHN!", + EOF, + ], + } + flow = Flow.from_file("tests/TestFanInGraph.flyde") + + in_q = flow.node.inputs["str"].queue + out_q = Queue() + flow.node.outputs["out"].connect(out_q) + + flow.run() + + for inp in test_case["inputs"]: + in_q.put(inp) -# def test_with_graph(self): -# test_case = { -# "inputs": ["John", EOF], -# "outputs": [ -# "JOHNJOHNJOHN", -# "JOHNJOHNJOHN", -# "HELLO, JOHN!HELLO, JOHN!HELLO, JOHN!", -# EOF, -# ], -# } -# flow = Flow.from_file("tests/TestFanInGraph.flyde") - -# in_q = flow.node.inputs["str"].queue -# out_q = Queue() -# flow.node.outputs["out"].connect(out_q) - -# flow.run() - -# for inp in test_case["inputs"]: -# in_q.put(inp) - -# # Get all outputs until EOF -# output_list = [] -# count = 0 -# limit = len(test_case["outputs"]) -# out = None -# while count < limit and out != EOF: -# out = out_q.get() -# output_list.append(out) -# count += 1 - -# # Compare expected and actual lists ignoring the order of elements -# self.assertCountEqual(test_case["outputs"], output_list) -# # EOF must be the last output -# self.assertEqual(EOF, output_list[-1]) - -# flow.stopped.wait() -# self.assertTrue(flow.stopped.is_set()) + # Get all outputs until EOF + output_list = [] + count = 0 + limit = len(test_case["outputs"]) + out = None + while count < limit and out != EOF: + out = out_q.get() + output_list.append(out) + count += 1 + + # Compare expected and actual lists ignoring the order of elements + self.assertCountEqual(test_case["outputs"], output_list) + # EOF must be the last output + self.assertEqual(EOF, output_list[-1]) + + flow.stopped.wait() + self.assertTrue(flow.stopped.is_set()) From 092c950e4244c909787ea42a418b7535db434653 Mon Sep 17 00:00:00 2001 From: Vladimir Sibirov Date: Sat, 12 Jul 2025 21:26:06 +0200 Subject: [PATCH 09/18] Preliminary implementation of `.flyde-nodes.json` support --- flyde/cli.py | 174 ++++++++++++++++- flyde/cli.pyi | 12 +- flyde/flow.pyi | 3 +- flyde/node.py | 2 +- tests/.flyde-nodes.json | 421 ++++++++++++++++++++++++++++++++++++++++ tests/test_cli.py | 416 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 1021 insertions(+), 7 deletions(-) create mode 100644 tests/.flyde-nodes.json create mode 100644 tests/test_cli.py diff --git a/flyde/cli.py b/flyde/cli.py index d7b8be4..bcaf016 100644 --- a/flyde/cli.py +++ b/flyde/cli.py @@ -1,14 +1,18 @@ #!/usr/bin/env python3 import argparse +import glob import importlib +import importlib.util +import json import logging import os import pprint +import sys from flyde.flow import Flow, add_folder_to_path -from flyde.node import Component +from flyde.node import SUPPORTED_MACROS, Component -log_level = getattr(logging, os.getenv('LOG_LEVEL', 'INFO').upper(), logging.INFO) +log_level = getattr(logging, os.getenv("LOG_LEVEL", "INFO").upper(), logging.INFO) logging.basicConfig(level=log_level) logger = logging.getLogger(__name__) @@ -17,6 +21,163 @@ def py_path_to_module(py_path: str) -> str: return py_path.replace("/", ".").replace(".py", "") +def convert_class_name_to_display_name(class_name: str) -> str: + """Convert a class name like 'MyCustomNode' to 'My Custom Node'.""" + # Insert space before uppercase letters that follow lowercase letters + import re + + return re.sub(r"(?<=[a-z])(?=[A-Z])", " ", class_name) + + +def is_stdlib_node(node_name: str) -> bool: + """Check if a node name matches a stdlib node.""" + return node_name in SUPPORTED_MACROS + + +def collect_components_from_directory(directory_path: str) -> dict: + """Collect all Component subclasses from .py files in a directory.""" + components = {} + + # Convert directory path to absolute path + abs_dir = os.path.abspath(directory_path) + + # Add the directory to Python path for imports + if abs_dir not in sys.path: + sys.path.insert(0, abs_dir) + + # Find all .py files in the directory + py_files = glob.glob(os.path.join(directory_path, "*.py")) + + for py_file in py_files: + # Skip __init__.py files + if os.path.basename(py_file) == "__init__.py": + continue + + try: + # Convert file path to module name + module_name = os.path.splitext(os.path.basename(py_file))[0] + + # Import the module + spec = importlib.util.spec_from_file_location(module_name, py_file) + if spec and spec.loader: + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + # Find all Component subclasses + for name in dir(module): + obj = getattr(module, name) + if ( + name != "Component" + and isinstance(obj, type) + and issubclass(obj, Component) + and obj.__module__ == module_name + ): + components[name] = obj + + except Exception as e: + logger.warning(f"Failed to import module from {py_file}: {e}") + + return components + + +def generate_node_json(node_name: str, component_class) -> dict: + """Generate JSON structure for a single component.""" + # Get description from docstring + description = (component_class.__doc__ or "").strip() + + # Generate display name + display_name = convert_class_name_to_display_name(node_name) + + # Determine source + if is_stdlib_node(node_name): + source = {"type": "package", "data": "@flyde/nodes"} + display_name = f"Overridden {display_name}" + description = f"This overrides the standard {node_name} node" + else: + source = {"type": "custom", "data": f"custom://{node_name}"} + + # Build inputs structure + inputs = {} + if hasattr(component_class, "inputs") and component_class.inputs: + for input_name, input_obj in component_class.inputs.items(): + inputs[input_name] = {"description": input_obj.description or f"{input_name} input"} + + # Build outputs structure + outputs = {} + if hasattr(component_class, "outputs") and component_class.outputs: + for output_name, output_obj in component_class.outputs.items(): + outputs[output_name] = {"description": output_obj.description or f"{output_name} output"} + + # Build the node structure + node_data = { + "id": node_name, + "type": "code", + "displayName": display_name, + "description": description, + "source": source, + "editorNode": { + "id": node_name, + "displayName": display_name, + "description": description, + "inputs": inputs, + "outputs": outputs, + "editorConfig": {"type": "structured"}, + }, + "config": {}, + } + + # Add icon for custom nodes + if not is_stdlib_node(node_name): + node_data["icon"] = "fa-solid fa-user" + node_data["editorNode"]["icon"] = "fa-solid fa-user" + + return node_data + + +def gen_json(directory_path: str): + """Generate JSON file for all components in a directory.""" + print(f"Generating JSON file for components in directory {directory_path}") + + # Collect all components + components = collect_components_from_directory(directory_path) + + if not components: + print(f"No Component subclasses found in directory {directory_path}") + return + + # Build nodes structure + nodes = {} + custom_nodes = [] + stdlib_nodes = [] + + for node_name, component_class in components.items(): + nodes[node_name] = generate_node_json(node_name, component_class) + + if is_stdlib_node(node_name): + stdlib_nodes.append(node_name) + else: + custom_nodes.append(node_name) + + # Build groups + groups = [] + if custom_nodes: + groups.append({"title": "Custom Runtime Nodes", "nodeIds": custom_nodes}) + if stdlib_nodes: + groups.append({"title": "Overridden Stdlib", "nodeIds": stdlib_nodes}) + + # Build final JSON structure + json_data = {"nodes": nodes, "groups": groups} + + # Write to file + output_file = os.path.join(directory_path, ".flyde-nodes.json") + with open(output_file, "w") as f: + json.dump(json_data, f, indent=2) + + print(f"Generated {output_file} with {len(components)} components") + print(f"Custom nodes: {custom_nodes}") + print(f"Stdlib overrides: {stdlib_nodes}") + + def gen(path: str): """Generate TypeScript files for a module.""" print(f"Generating TypeScript files for module {path}") @@ -55,7 +216,7 @@ def main(): parser.add_argument( "path", type=str, - help='Path to a ".flyde" flow file to run or a Python ".py" module to generate typescript definitions for', + help='Path to a ".flyde" flow file to run, a Python ".py" module, or a directory to generate definitions for', ) args = parser.parse_args() @@ -75,6 +236,11 @@ def main(): add_folder_to_path(args.path) # Add current folder to path when resolving modules relative to the current folder add_folder_to_path(".") - gen(args.path) + + # Check if path is a directory or file + if os.path.isdir(args.path): + gen_json(args.path) + else: + gen(args.path) else: raise ValueError(f"Unknown command: {args.command}") diff --git a/flyde/cli.pyi b/flyde/cli.pyi index 537a5ce..ea8d23b 100644 --- a/flyde/cli.pyi +++ b/flyde/cli.pyi @@ -1,11 +1,21 @@ from _typeshed import Incomplete from flyde.flow import Flow as Flow, add_folder_to_path as add_folder_to_path -from flyde.node import Component as Component +from flyde.node import Component as Component, SUPPORTED_MACROS as SUPPORTED_MACROS log_level: Incomplete logger: Incomplete def py_path_to_module(py_path: str) -> str: ... +def convert_class_name_to_display_name(class_name: str) -> str: + """Convert a class name like 'MyCustomNode' to 'My Custom Node'.""" +def is_stdlib_node(node_name: str) -> bool: + """Check if a node name matches a stdlib node.""" +def collect_components_from_directory(directory_path: str) -> dict: + """Collect all Component subclasses from .py files in a directory.""" +def generate_node_json(node_name: str, component_class) -> dict: + """Generate JSON structure for a single component.""" +def gen_json(directory_path: str): + """Generate JSON file for all components in a directory.""" def gen(path: str): """Generate TypeScript files for a module.""" def main() -> None: ... diff --git a/flyde/flow.pyi b/flyde/flow.pyi index 27ac0fd..14e2b0b 100644 --- a/flyde/flow.pyi +++ b/flyde/flow.pyi @@ -23,7 +23,8 @@ class Flow: def factory(self, class_name: str, args: InstanceArgs): """Factory method to create a node from a class name and arguments. - It is used by the runtime to create nodes from the YAML definition or on the fly.""" + It is used by the runtime to create nodes from the YAML definition or on the fly. + """ def run(self) -> None: """Start the flow running. This is a non-blocking call as the flow runs in a separate thread.""" def run_sync(self) -> None: diff --git a/flyde/node.py b/flyde/node.py index 4b6a146..40b1754 100644 --- a/flyde/node.py +++ b/flyde/node.py @@ -11,7 +11,7 @@ logger = logging.getLogger(__name__) -SUPPORTED_MACROS = ["InlineValue", "Conditional", "GetAttribute"] +SUPPORTED_MACROS = ["InlineValue", "Conditional", "GetAttribute", "Http"] class InstanceType(Enum): diff --git a/tests/.flyde-nodes.json b/tests/.flyde-nodes.json new file mode 100644 index 0000000..5ca819e --- /dev/null +++ b/tests/.flyde-nodes.json @@ -0,0 +1,421 @@ +{ + "nodes": { + "Capitalize": { + "id": "Capitalize", + "type": "code", + "displayName": "Capitalize", + "description": "A component that capitalizes the input.", + "source": { + "type": "custom", + "data": "custom://Capitalize" + }, + "editorNode": { + "id": "Capitalize", + "displayName": "Capitalize", + "description": "A component that capitalizes the input.", + "inputs": { + "inp": { + "description": "The input" + } + }, + "outputs": { + "out": { + "description": "The output" + } + }, + "editorConfig": { + "type": "structured" + }, + "icon": "fa-solid fa-user" + }, + "config": {}, + "icon": "fa-solid fa-user" + }, + "Echo": { + "id": "Echo", + "type": "code", + "displayName": "Echo", + "description": "A simple component that echoes the input.", + "source": { + "type": "custom", + "data": "custom://Echo" + }, + "editorNode": { + "id": "Echo", + "displayName": "Echo", + "description": "A simple component that echoes the input.", + "inputs": { + "inp": { + "description": "The input" + } + }, + "outputs": { + "out": { + "description": "The output" + } + }, + "editorConfig": { + "type": "structured" + }, + "icon": "fa-solid fa-user" + }, + "config": {}, + "icon": "fa-solid fa-user" + }, + "Format": { + "id": "Format", + "type": "code", + "displayName": "Format", + "description": "Formats the input value with a given format string and sends it to out.", + "source": { + "type": "custom", + "data": "custom://Format" + }, + "editorNode": { + "id": "Format", + "displayName": "Format", + "description": "Formats the input value with a given format string and sends it to out.", + "inputs": { + "inp": { + "description": "The input" + }, + "format": { + "description": "The format string" + } + }, + "outputs": { + "out": { + "description": "The formatted output" + } + }, + "editorConfig": { + "type": "structured" + }, + "icon": "fa-solid fa-user" + }, + "config": {}, + "icon": "fa-solid fa-user" + }, + "RepeatWordNTimes": { + "id": "RepeatWordNTimes", + "type": "code", + "displayName": "Repeat Word NTimes", + "description": "A component that has both inputs and outputs and a sticky input.", + "source": { + "type": "custom", + "data": "custom://RepeatWordNTimes" + }, + "editorNode": { + "id": "RepeatWordNTimes", + "displayName": "Repeat Word NTimes", + "description": "A component that has both inputs and outputs and a sticky input.", + "inputs": { + "word": { + "description": "The input" + }, + "times": { + "description": "The number of times to repeat the input" + } + }, + "outputs": { + "out": { + "description": "The output" + } + }, + "editorConfig": { + "type": "structured" + }, + "icon": "fa-solid fa-user" + }, + "config": {}, + "icon": "fa-solid fa-user" + }, + "CustomBob": { + "id": "CustomBob", + "type": "code", + "displayName": "Custom Bob", + "description": "A custom external node named Bob", + "source": { + "type": "custom", + "data": "custom://CustomBob" + }, + "editorNode": { + "id": "CustomBob", + "displayName": "Custom Bob", + "description": "A custom external node named Bob", + "inputs": { + "value": { + "description": "Input value to process" + } + }, + "outputs": { + "result": { + "description": "Processed result from external runtime" + } + }, + "editorConfig": { + "type": "structured" + }, + "icon": "fa-solid fa-user" + }, + "config": {}, + "icon": "fa-solid fa-user" + }, + "InlineValue": { + "id": "InlineValue", + "type": "code", + "displayName": "Overridden Inline Value", + "description": "This overrides the standard InlineValue node", + "source": { + "type": "package", + "data": "@flyde/nodes" + }, + "editorNode": { + "id": "InlineValue", + "displayName": "Overridden Inline Value", + "description": "This overrides the standard InlineValue node", + "inputs": {}, + "outputs": { + "value": { + "description": "The overridden value" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "TestCustomComponent": { + "id": "TestCustomComponent", + "type": "code", + "displayName": "Test Custom Component", + "description": "A test component for unit testing.", + "source": { + "type": "custom", + "data": "custom://TestCustomComponent" + }, + "editorNode": { + "id": "TestCustomComponent", + "displayName": "Test Custom Component", + "description": "A test component for unit testing.", + "inputs": { + "input1": { + "description": "First test input" + }, + "input2": { + "description": "Second test input" + } + }, + "outputs": { + "output": { + "description": "Test output" + } + }, + "editorConfig": { + "type": "structured" + }, + "icon": "fa-solid fa-user" + }, + "config": {}, + "icon": "fa-solid fa-user" + }, + "AllStickyInputsComponent": { + "id": "AllStickyInputsComponent", + "type": "code", + "displayName": "All Sticky Inputs Component", + "description": "A component with only sticky inputs to test single execution.", + "source": { + "type": "custom", + "data": "custom://AllStickyInputsComponent" + }, + "editorNode": { + "id": "AllStickyInputsComponent", + "displayName": "All Sticky Inputs Component", + "description": "A component with only sticky inputs to test single execution.", + "inputs": { + "a": { + "description": "First sticky input" + }, + "b": { + "description": "Second sticky input" + } + }, + "outputs": { + "result": { + "description": "Result of operation" + } + }, + "editorConfig": { + "type": "structured" + }, + "icon": "fa-solid fa-user" + }, + "config": {}, + "icon": "fa-solid fa-user" + }, + "CustomRunComponent": { + "id": "CustomRunComponent", + "type": "code", + "displayName": "Custom Run Component", + "description": "A component that has a custom run and shutdown handlers.", + "source": { + "type": "custom", + "data": "custom://CustomRunComponent" + }, + "editorNode": { + "id": "CustomRunComponent", + "displayName": "Custom Run Component", + "description": "A component that has a custom run and shutdown handlers.", + "inputs": { + "s": { + "description": "Individual strings" + } + }, + "outputs": { + "l": { + "description": "List of strings" + } + }, + "editorConfig": { + "type": "structured" + }, + "icon": "fa-solid fa-user" + }, + "config": {}, + "icon": "fa-solid fa-user" + }, + "InvalidSendProcess": { + "id": "InvalidSendProcess", + "type": "code", + "displayName": "Invalid Send Process", + "description": "A component that tries to send a message without a corresponding output.", + "source": { + "type": "custom", + "data": "custom://InvalidSendProcess" + }, + "editorNode": { + "id": "InvalidSendProcess", + "displayName": "Invalid Send Process", + "description": "A component that tries to send a message without a corresponding output.", + "inputs": { + "s": { + "description": "Individual strings" + } + }, + "outputs": {}, + "editorConfig": { + "type": "structured" + }, + "icon": "fa-solid fa-user" + }, + "config": {}, + "icon": "fa-solid fa-user" + }, + "NoProcessComponent": { + "id": "NoProcessComponent", + "type": "code", + "displayName": "No Process Component", + "description": "A component to test no inputs, outputs, and no process method.", + "source": { + "type": "custom", + "data": "custom://NoProcessComponent" + }, + "editorNode": { + "id": "NoProcessComponent", + "displayName": "No Process Component", + "description": "A component to test no inputs, outputs, and no process method.", + "inputs": {}, + "outputs": {}, + "editorConfig": { + "type": "structured" + }, + "icon": "fa-solid fa-user" + }, + "config": {}, + "icon": "fa-solid fa-user" + }, + "SinkComponent": { + "id": "SinkComponent", + "type": "code", + "displayName": "Sink Component", + "description": "A component that only has inputs.", + "source": { + "type": "custom", + "data": "custom://SinkComponent" + }, + "editorNode": { + "id": "SinkComponent", + "displayName": "Sink Component", + "description": "A component that only has inputs.", + "inputs": { + "word": { + "description": "The input" + }, + "output": { + "description": "Object to store result in" + } + }, + "outputs": {}, + "editorConfig": { + "type": "structured" + }, + "icon": "fa-solid fa-user" + }, + "config": {}, + "icon": "fa-solid fa-user" + }, + "SourceComponent": { + "id": "SourceComponent", + "type": "code", + "displayName": "Source Component", + "description": "A component that only has outputs.", + "source": { + "type": "custom", + "data": "custom://SourceComponent" + }, + "editorNode": { + "id": "SourceComponent", + "displayName": "Source Component", + "description": "A component that only has outputs.", + "inputs": {}, + "outputs": { + "out": { + "description": "The output" + } + }, + "editorConfig": { + "type": "structured" + }, + "icon": "fa-solid fa-user" + }, + "config": {}, + "icon": "fa-solid fa-user" + } + }, + "groups": [ + { + "title": "Custom Runtime Nodes", + "nodeIds": [ + "Capitalize", + "Echo", + "Format", + "RepeatWordNTimes", + "CustomBob", + "TestCustomComponent", + "AllStickyInputsComponent", + "CustomRunComponent", + "InvalidSendProcess", + "NoProcessComponent", + "SinkComponent", + "SourceComponent" + ] + }, + { + "title": "Overridden Stdlib", + "nodeIds": [ + "InlineValue" + ] + } + ] +} \ No newline at end of file diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..ea6d9ed --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,416 @@ +import json +import os +import tempfile +import unittest +from unittest.mock import patch + +from flyde.cli import ( + collect_components_from_directory, + convert_class_name_to_display_name, + gen_json, + generate_node_json, + is_stdlib_node, +) +from flyde.io import Input, Output +from flyde.node import Component + + +class TestCLIHelpers(unittest.TestCase): + def test_convert_class_name_to_display_name(self): + test_cases = [ + ("MyCustomNode", "My Custom Node"), + ("HTTPClient", "HTTPClient"), # Adjacent capitals stay together + ("XMLParser", "XMLParser"), + ("SimpleNode", "Simple Node"), + ("Node", "Node"), # Single word + ("MyHTTPClient", "My HTTPClient"), + ("CustomBob", "Custom Bob"), + ("CustomAlice", "Custom Alice"), + ] + + for class_name, expected in test_cases: + with self.subTest(class_name=class_name): + result = convert_class_name_to_display_name(class_name) + self.assertEqual(result, expected) + + def test_is_stdlib_node(self): + from flyde.node import SUPPORTED_MACROS + + # Test that all supported macros are detected as stdlib nodes + for macro in SUPPORTED_MACROS: + with self.subTest(node_name=macro): + result = is_stdlib_node(macro) + self.assertTrue(result) + + # Test that custom nodes are not detected as stdlib nodes + custom_nodes = ["CustomNode", "MyComponent", "Echo", "Format"] + for node_name in custom_nodes: + with self.subTest(node_name=node_name): + result = is_stdlib_node(node_name) + self.assertFalse(result) + + +class TestCustomComponent(Component): + """A test component for unit testing.""" + + inputs = { + "input1": Input(description="First test input"), + "input2": Input(description="Second test input"), + } + outputs = { + "output": Output(description="Test output"), + } + + def process(self, input1, input2): + return {"output": f"{input1}-{input2}"} + + +class CustomBob(Component): + """A custom external node named Bob""" + + inputs = { + "value": Input(description="Input value to process"), + } + outputs = { + "result": Output(description="Processed result from external runtime"), + } + + def process(self, value): + return {"result": f"Bob processed: {value}"} + + +class InlineValue(Component): + """This overrides the standard InlineValue node""" + + outputs = { + "value": Output(description="The overridden value"), + } + + def process(self): + return {"value": "overridden"} + + +class TestGenerateNodeJson(unittest.TestCase): + def test_generate_custom_node_json(self): + result = generate_node_json("CustomBob", CustomBob) + + expected = { + "id": "CustomBob", + "type": "code", + "displayName": "Custom Bob", + "description": "A custom external node named Bob", + "icon": "fa-solid fa-user", + "source": {"type": "custom", "data": "custom://CustomBob"}, + "editorNode": { + "id": "CustomBob", + "displayName": "Custom Bob", + "description": "A custom external node named Bob", + "icon": "fa-solid fa-user", + "inputs": {"value": {"description": "Input value to process"}}, + "outputs": {"result": {"description": "Processed result from external runtime"}}, + "editorConfig": {"type": "structured"}, + }, + "config": {}, + } + + self.assertEqual(result, expected) + + def test_generate_stdlib_override_json(self): + result = generate_node_json("InlineValue", InlineValue) + + expected = { + "id": "InlineValue", + "type": "code", + "displayName": "Overridden Inline Value", + "description": "This overrides the standard InlineValue node", + "source": {"type": "package", "data": "@flyde/nodes"}, + "editorNode": { + "id": "InlineValue", + "displayName": "Overridden Inline Value", + "description": "This overrides the standard InlineValue node", + "inputs": {}, + "outputs": {"value": {"description": "The overridden value"}}, + "editorConfig": {"type": "structured"}, + }, + "config": {}, + } + + self.assertEqual(result, expected) + + def test_generate_node_with_no_docstring(self): + class NodeWithoutDoc(Component): + inputs = {"inp": Input(description="Input")} + outputs = {"out": Output(description="Output")} + + result = generate_node_json("NodeWithoutDoc", NodeWithoutDoc) + + self.assertEqual(result["description"], "") + self.assertEqual(result["editorNode"]["description"], "") + + def test_generate_node_with_no_inputs_outputs(self): + class EmptyNode(Component): + """An empty node""" + + result = generate_node_json("EmptyNode", EmptyNode) + + self.assertEqual(result["editorNode"]["inputs"], {}) + self.assertEqual(result["editorNode"]["outputs"], {}) + + +class TestCollectComponents(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + import shutil + + shutil.rmtree(self.temp_dir) + + def test_collect_components_from_directory(self): + # Create test Python files + test_component_py = ''' +from flyde.io import Input, Output +from flyde.node import Component + +class TestNode1(Component): + """First test node""" + inputs = {"inp": Input(description="Input")} + outputs = {"out": Output(description="Output")} + +class TestNode2(Component): + """Second test node""" + inputs = {"data": Input(description="Data input")} + outputs = {"result": Output(description="Result output")} + +# This should be ignored +class NotAComponent: + pass +''' + + another_component_py = ''' +from flyde.io import Input, Output +from flyde.node import Component + +class AnotherNode(Component): + """Another test node""" + outputs = {"value": Output(description="Value output")} +''' + + # Write test files + with open(os.path.join(self.temp_dir, "test_components.py"), "w") as f: + f.write(test_component_py) + + with open(os.path.join(self.temp_dir, "another.py"), "w") as f: + f.write(another_component_py) + + # Create __init__.py (should be ignored) + with open(os.path.join(self.temp_dir, "__init__.py"), "w") as f: + f.write("# This should be ignored") + + # Collect components + components = collect_components_from_directory(self.temp_dir) + + # Should find 3 components + self.assertEqual(len(components), 3) + self.assertIn("TestNode1", components) + self.assertIn("TestNode2", components) + self.assertIn("AnotherNode", components) + + # Check that components are actually Component subclasses + for name, cls in components.items(): + self.assertTrue(issubclass(cls, Component)) + + def test_collect_components_invalid_syntax(self): + # Create a file with invalid Python syntax + invalid_py = """ +This is not valid Python syntax! +class InvalidNode(Component: + pass +""" + + with open(os.path.join(self.temp_dir, "invalid.py"), "w") as f: + f.write(invalid_py) + + # Should handle the error gracefully + components = collect_components_from_directory(self.temp_dir) + self.assertEqual(len(components), 0) + + def test_collect_components_empty_directory(self): + # Test with empty directory + components = collect_components_from_directory(self.temp_dir) + self.assertEqual(len(components), 0) + + +class TestGenJson(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + import shutil + + shutil.rmtree(self.temp_dir) + + def test_gen_json_with_mixed_components(self): + # Create test files with both custom and stdlib override components + components_py = ''' +from flyde.io import Input, Output +from flyde.node import Component + +class CustomBob(Component): + """A custom external node named Bob""" + inputs = {"value": Input(description="Input value to process")} + outputs = {"result": Output(description="Processed result from external runtime")} + +class CustomAlice(Component): + """Another custom external node""" + inputs = { + "input1": Input(description="First input"), + "input2": Input(description="Second input") + } + outputs = {"output": Output(description="Combined output")} + +class InlineValue(Component): + """This overrides the standard InlineValue node""" + outputs = {"value": Output(description="The overridden value")} +''' + + with open(os.path.join(self.temp_dir, "components.py"), "w") as f: + f.write(components_py) + + # Generate JSON + gen_json(self.temp_dir) + + # Check that the file was created + output_file = os.path.join(self.temp_dir, ".flyde-nodes.json") + self.assertTrue(os.path.exists(output_file)) + + # Load and verify the JSON content + with open(output_file, "r") as f: + data = json.load(f) + + # Check structure + self.assertIn("nodes", data) + self.assertIn("groups", data) + + # Check nodes + nodes = data["nodes"] + self.assertEqual(len(nodes), 3) + self.assertIn("CustomBob", nodes) + self.assertIn("CustomAlice", nodes) + self.assertIn("InlineValue", nodes) + + # Check CustomBob + bob = nodes["CustomBob"] + self.assertEqual(bob["id"], "CustomBob") + self.assertEqual(bob["displayName"], "Custom Bob") + self.assertEqual(bob["source"]["type"], "custom") + self.assertEqual(bob["source"]["data"], "custom://CustomBob") + self.assertIn("icon", bob) + + # Check InlineValue (stdlib override) + inline = nodes["InlineValue"] + self.assertEqual(inline["id"], "InlineValue") + self.assertEqual(inline["displayName"], "Overridden Inline Value") + self.assertEqual(inline["source"]["type"], "package") + self.assertEqual(inline["source"]["data"], "@flyde/nodes") + self.assertNotIn("icon", inline) + + # Check groups + groups = data["groups"] + self.assertEqual(len(groups), 2) + + # Find groups by title + custom_group = next(g for g in groups if g["title"] == "Custom Runtime Nodes") + stdlib_group = next(g for g in groups if g["title"] == "Overridden Stdlib") + + self.assertCountEqual(custom_group["nodeIds"], ["CustomBob", "CustomAlice"]) + self.assertCountEqual(stdlib_group["nodeIds"], ["InlineValue"]) + + def test_gen_json_empty_directory(self): + # Test with directory containing no components + empty_py = """ +# This file has no Component subclasses +def some_function(): + pass + +class NotAComponent: + pass +""" + + with open(os.path.join(self.temp_dir, "empty.py"), "w") as f: + f.write(empty_py) + + # Capture stdout to verify the message + with patch("builtins.print") as mock_print: + gen_json(self.temp_dir) + mock_print.assert_any_call(f"No Component subclasses found in directory {self.temp_dir}") + + # Should not create the JSON file + output_file = os.path.join(self.temp_dir, ".flyde-nodes.json") + self.assertFalse(os.path.exists(output_file)) + + def test_gen_json_only_custom_nodes(self): + # Test with only custom nodes (no stdlib overrides) + components_py = ''' +from flyde.io import Input, Output +from flyde.node import Component + +class CustomNode1(Component): + """First custom node""" + inputs = {"inp": Input(description="Input")} + outputs = {"out": Output(description="Output")} + +class CustomNode2(Component): + """Second custom node""" + outputs = {"result": Output(description="Result")} +''' + + with open(os.path.join(self.temp_dir, "components.py"), "w") as f: + f.write(components_py) + + gen_json(self.temp_dir) + + output_file = os.path.join(self.temp_dir, ".flyde-nodes.json") + with open(output_file, "r") as f: + data = json.load(f) + + # Should have one group for custom nodes only + groups = data["groups"] + self.assertEqual(len(groups), 1) + self.assertEqual(groups[0]["title"], "Custom Runtime Nodes") + self.assertCountEqual(groups[0]["nodeIds"], ["CustomNode1", "CustomNode2"]) + + # Ensure no stdlib overrides group exists + for group in groups: + self.assertNotEqual(group["title"], "Overridden Stdlib") + + def test_gen_json_only_stdlib_overrides(self): + # Test with only stdlib override nodes + components_py = ''' +from flyde.io import Input, Output +from flyde.node import Component + +class InlineValue(Component): + """Override InlineValue""" + outputs = {"value": Output(description="Value")} + +class Conditional(Component): + """Override Conditional""" + inputs = {"condition": Input(description="Condition")} + outputs = {"result": Output(description="Result")} +''' + + with open(os.path.join(self.temp_dir, "components.py"), "w") as f: + f.write(components_py) + + gen_json(self.temp_dir) + + output_file = os.path.join(self.temp_dir, ".flyde-nodes.json") + with open(output_file, "r") as f: + data = json.load(f) + + # Should have one group for stdlib overrides only + groups = data["groups"] + self.assertEqual(len(groups), 1) + self.assertEqual(groups[0]["title"], "Overridden Stdlib") + self.assertCountEqual(groups[0]["nodeIds"], ["InlineValue", "Conditional"]) From 33a3c283daffda15827c741ff260647599baa633 Mon Sep 17 00:00:00 2001 From: Vladimir Sibirov Date: Sun, 13 Jul 2025 21:47:46 +0200 Subject: [PATCH 10/18] Improved `.flyde-nodes.json` support, removed .ts stubs --- Makefile | 14 +++--- README.md | 6 +-- docs/quickstart.md | 16 ++++++- docs/usage.md | 21 ++++++--- examples/Clustering.flyde | 28 ++++++------ examples/HelloPy.flyde | 4 +- examples/mylib/components.flyde.ts | 25 ----------- examples/mylib/dataframe.flyde.ts | 26 ------------ examples/mylib/kmeans.flyde.ts | 52 ----------------------- flyde/cli.py | 54 +++++++++--------------- flyde/cli.pyi | 4 +- flyde/flow.py | 66 ++++++++++++++++++++++++++--- flyde/node.py | 39 ++--------------- flyde/node.pyi | 7 ++- tests/.flyde-nodes.json | 24 +++++------ tests/Repeat3Times.flyde | 8 ++-- tests/TestFanIn.flyde | 12 +++--- tests/TestFanInGraph.flyde | 8 ++-- tests/TestInOutFlow.flyde | 8 ++-- tests/TestIsolatedFlow.flyde | 4 +- tests/TestNestedFlow.flyde | 8 ++-- tests/components.flyde.ts | 52 ----------------------- tests/test_cli.py | 28 ++++++++---- tests/test_flow.py | 68 ++++++++++++++++++++++++++++++ tests/test_node.py | 22 ---------- 25 files changed, 266 insertions(+), 338 deletions(-) delete mode 100644 examples/mylib/components.flyde.ts delete mode 100644 examples/mylib/dataframe.flyde.ts delete mode 100644 examples/mylib/kmeans.flyde.ts delete mode 100644 tests/components.flyde.ts diff --git a/Makefile b/Makefile index 5bfeffb..4266a31 100644 --- a/Makefile +++ b/Makefile @@ -7,14 +7,12 @@ SRC_DIR = flyde TEST_DIR = tests # Targets -.PHONY: ts test cover +.PHONY: gen test cover -ts: - @echo "Building the project..." - # For each *.py file in the examples/mylib directory run the ./flyde.py gen command to generate TS bindings - @for file in $(LIB_DIR)/*.py; do \ - ./flyde.py gen $$file; \ - done +gen: + @echo "Generating component definitions..." + # Generate JSON definitions for the examples/mylib directory + @./pyflyde gen $(LIB_DIR)/ lint: @echo "Running linters..." @@ -46,7 +44,7 @@ builddist: @rm -f ./dist/* @$(PYTHON) -m build; -release: lint test stubgen builddist +release: lint test stubgen gen builddist @echo "Releasing the project..."; upload: diff --git a/README.md b/README.md index df07c55..d5f860a 100644 --- a/README.md +++ b/README.md @@ -61,13 +61,13 @@ Install Flyde VSCode extension from the [marketplace](https://marketplace.visual You can browse the component library in the panel on the right. To see your local components click the "View all" button. They will appear under the "Current project". Note that PyFlyde doesn't implement all of the Flyde's stdlib components, only a few essential ones. -Whenever you change your component library classes or their interfaces, use `pyflyde gen` command to generate `.flyde.ts` definitions, e.g.: +Whenever you change your component library classes or their interfaces, use `pyflyde gen` command to generate `.flyde-nodes.json` definitions, e.g.: ```bash -pyflyde gen examples/mylib/components.py +pyflyde gen examples/ ``` -Flyde editor needs `.flyde.ts` files in order to "see" your components. +This will recursively scan all Python files in the directory and its subdirectories to find PyFlyde components and generate a `.flyde-nodes.json` file with relative paths. Flyde editor needs `.flyde-nodes.json` files in order to "see" your components. ### Running a Machine Learning example and creating your first project diff --git a/docs/quickstart.md b/docs/quickstart.md index b08b611..67640af 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -42,7 +42,15 @@ pip install examples/ ## Running the Hello World example -Run the example flow: +First, generate the component metadata for the examples: + +```bash +pyflyde gen examples/ +``` + +This will recursively scan all Python files in the `examples/` directory and generate a `.flyde-nodes.json` file with metadata for all PyFlyde components found. + +Then run the example flow: ```bash pyflyde examples/HelloPy.flyde @@ -54,6 +62,12 @@ It should print "Hello Flyde!" in the console. `examples/Clustering.flyde` is a more complex example which uses Pandas and Scikit-Learn to run K-means clustering on a [wine clustering dataset from Kaggle](https://www.kaggle.com/harrywang/wine-dataset-for-clustering). It's a PyFlyde version of https://github.com/Shivangi0503/Wine_Clustering_KMeans. +The component metadata should already be generated from the previous step, but if you add new components, remember to run: + +```bash +pyflyde gen examples/ +``` + Open the `examples/Clustering.flyde` in Flyde VSCode visual editor to see how it looks like. To run this example, use the `pyflyde` command line tool: diff --git a/docs/usage.md b/docs/usage.md index e9c3fc6..e507010 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -14,16 +14,25 @@ You can omit the `run` command because it is the default one. The following comm pyflyde examples/HelloWorld.flyde ``` -## Generating TS definitions for Flyde visual editor +## Generating component definitions for Flyde visual editor -Flyde visual editor is written for TypeScript runtime and is not aware of your Python nodes. To make your local nodes appear in the Flyde editor, you need to generate `.flyde.ts` files for them. +To make your Python nodes appear in the Flyde visual editor, you need to generate `.flyde-nodes.json` metadata files for them. -For example: +Generate JSON definitions for a directory: ```bash -pyflyde gen mypackage/supermodule.py +pyflyde gen mypackage/ ``` -will generate `mypackage/supermodule.flyde.ts` TypeScript defintions for Flyde using the contents of your `mypackage.submodule` module. +This will recursively scan all `.py` files in the directory and its subdirectories, then generate a `.flyde-nodes.json` file in the specified directory containing metadata for all PyFlyde components found. The paths in the generated JSON file are relative to the directory containing the `.flyde-nodes.json` file, making the component library portable. + +For example, if you have components in: +- `mypackage/components.py` +- `mypackage/utils/helpers.py` + +The generated `.flyde-nodes.json` will reference them as: +- `custom://components.py/ComponentName` +- `custom://utils/helpers.py/HelperComponentName` + +You should run `pyflyde gen` every time you create new modules containing PyFlyde nodes or whenever you update node signatures (name, description, inputs, outputs, etc.). -You should run `pyflyde gen` every time your create new modules containing PyFlyde nodes or whenever you update node signature (name, description, inputs, outputs, etc.). diff --git a/examples/Clustering.flyde b/examples/Clustering.flyde index ec4b952..12c928a 100644 --- a/examples/Clustering.flyde +++ b/examples/Clustering.flyde @@ -13,8 +13,8 @@ node: value: "{{file_path}}" type: code source: - type: file - data: mylib/dataframe.flyde.ts + type: custom + data: custom://mylib/dataframe.py/LoadDataset - pos: x: -939.2435131835937 y: 61.415870666503906 @@ -47,8 +47,8 @@ node: value: "{{dataframe}}" type: code source: - type: file - data: mylib/dataframe.flyde.ts + type: custom + data: custom://mylib/dataframe.py/Scale - pos: x: -24.11493484497072 y: 62.353389739990234 @@ -64,8 +64,8 @@ node: value: "{{max_clusters}}" type: code source: - type: file - data: mylib/kmeans.flyde.ts + type: custom + data: custom://mylib/kmeans.py/KMeansNClusters - pos: x: 234.1243896484375 y: 185.80514907836914 @@ -81,8 +81,8 @@ node: value: "{{n_clusters}}" type: code source: - type: file - data: mylib/kmeans.flyde.ts + type: custom + data: custom://mylib/kmeans.py/KMeansCluster - pos: x: -431.2221974563599 y: -46.37674593925476 @@ -115,8 +115,8 @@ node: value: "{{scaled_dataframe}}" type: code source: - type: file - data: mylib/kmeans.flyde.ts + type: custom + data: custom://mylib/kmeans.py/PCA2 - pos: x: 1010.514711303711 y: 252.2306843566895 @@ -135,8 +135,8 @@ node: value: "{{kmeans_result}}" type: code source: - type: file - data: mylib/kmeans.flyde.ts + type: custom + data: custom://mylib/kmeans.py/Visualize - pos: x: 736.7720812988282 y: 149.70720727920536 @@ -149,8 +149,8 @@ node: value: "{{scaled_dataframe}}" type: code source: - type: file - data: mylib/kmeans.flyde.ts + type: custom + data: custom://mylib/kmeans.py/PCA2 - pos: x: 513.1246850585937 y: 141.11043981841965 diff --git a/examples/HelloPy.flyde b/examples/HelloPy.flyde index d0fa0a7..802cc67 100644 --- a/examples/HelloPy.flyde +++ b/examples/HelloPy.flyde @@ -18,8 +18,8 @@ node: value: Hello, Flyde! type: code source: - type: file - data: mylib/components.flyde.ts + type: custom + data: custom://mylib/components.py/Print connections: [] id: Example inputs: {} diff --git a/examples/mylib/components.flyde.ts b/examples/mylib/components.flyde.ts deleted file mode 100644 index 20b73a8..0000000 --- a/examples/mylib/components.flyde.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { CodeNode } from "@flyde/core"; - -export const Print: CodeNode = { - id: "Print", - description: "Prints the input message to the console.", - inputs: { - msg: {"description": "The message to print"} - }, - outputs: { }, - run: () => { return; }, -}; - -export const Concat: CodeNode = { - id: "Concat", - description: "Concatenates two strings.", - inputs: { - a: {"description": "The first string"}, - b: {"description": "The second string"} - }, - outputs: { - out: {"description": "The concatenated string"} - }, - run: () => { return; }, -}; - diff --git a/examples/mylib/dataframe.flyde.ts b/examples/mylib/dataframe.flyde.ts deleted file mode 100644 index 3670e17..0000000 --- a/examples/mylib/dataframe.flyde.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { CodeNode } from "@flyde/core"; - -export const LoadDataset: CodeNode = { - id: "LoadDataset", - description: "Loads a dataset from a file into a DataFrame.", - inputs: { - file_path: { description: "The path to the file containing the dataset" } - }, - outputs: { - dataframe: { description: "The loaded dataframe" } - }, - run: () => { return; }, -}; - -export const Scale: CodeNode = { - id: "Scale", - description: "Scales the features of a dataframe with a scikit-learn StandardScaler.", - inputs: { - dataframe: { description: "The dataframe to scale" } - }, - outputs: { - scaled_dataframe: { description: "The scaled dataframe" } - }, - run: () => { return; }, -}; - diff --git a/examples/mylib/kmeans.flyde.ts b/examples/mylib/kmeans.flyde.ts deleted file mode 100644 index 40a91f7..0000000 --- a/examples/mylib/kmeans.flyde.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { CodeNode } from "@flyde/core"; - -export const PCA2: CodeNode = { - id: "PCA2", - description: "Performs PCA on a dataframe and returns the first two principal components.", - inputs: { - scaled_dataframe: {"description": "The scaled dataframe to reduce"} - }, - outputs: { - pca_components: {"description": "The first two principal components"} - }, - run: () => { return; }, -}; - -export const KMeansNClusters: CodeNode = { - id: "KMeansNClusters", - description: "Finds the optimal number of clusters for K-means clustering using silhouette method.", - inputs: { - scaled_dataframe: {"description": "The scaled dataframe to cluster"}, - max_clusters: {"description": "The maximum number of clusters to consider"} - }, - outputs: { - n_clusters: {"description": "The optimal number of clusters"} - }, - run: () => { return; }, -}; - -export const KMeansCluster: CodeNode = { - id: "KMeansCluster", - description: "Clusters the dataframe using K-means clustering.", - inputs: { - scaled_dataframe: {"description": "The scaled dataframe to cluster"}, - n_clusters: {"description": "The number of clusters"} - }, - outputs: { - kmeans_result: {"description": "K-means clustering result"} - }, - run: () => { return; }, -}; - -export const Visualize: CodeNode = { - id: "Visualize", - description: "Visualizes the clustered dataframe.", - inputs: { - pca_components: {"description": "The first two principal components"}, - pca_centroids: {"description": "The centroids in PCA space"}, - kmeans_result: {"description": "K-means clustering result"} - }, - outputs: { }, - run: () => { return; }, -}; - diff --git a/flyde/cli.py b/flyde/cli.py index bcaf016..d401e63 100644 --- a/flyde/cli.py +++ b/flyde/cli.py @@ -35,7 +35,7 @@ def is_stdlib_node(node_name: str) -> bool: def collect_components_from_directory(directory_path: str) -> dict: - """Collect all Component subclasses from .py files in a directory.""" + """Collect all Component subclasses from .py files in a directory and its subdirectories.""" components = {} # Convert directory path to absolute path @@ -45,8 +45,8 @@ def collect_components_from_directory(directory_path: str) -> dict: if abs_dir not in sys.path: sys.path.insert(0, abs_dir) - # Find all .py files in the directory - py_files = glob.glob(os.path.join(directory_path, "*.py")) + # Find all .py files in the directory and subdirectories recursively + py_files = glob.glob(os.path.join(directory_path, "**", "*.py"), recursive=True) for py_file in py_files: # Skip __init__.py files @@ -54,11 +54,14 @@ def collect_components_from_directory(directory_path: str) -> dict: continue try: - # Convert file path to module name - module_name = os.path.splitext(os.path.basename(py_file))[0] + # Get relative path from the directory + relative_path = os.path.relpath(py_file, directory_path) + + # Convert file path to module name (handle subdirectories) + module_path = relative_path.replace(os.path.sep, ".").replace(".py", "") # Import the module - spec = importlib.util.spec_from_file_location(module_name, py_file) + spec = importlib.util.spec_from_file_location(module_path, py_file) if spec and spec.loader: module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) @@ -70,9 +73,9 @@ def collect_components_from_directory(directory_path: str) -> dict: name != "Component" and isinstance(obj, type) and issubclass(obj, Component) - and obj.__module__ == module_name + and obj.__module__ == module_path ): - components[name] = obj + components[name] = {"class": obj, "file_path": relative_path} except Exception as e: logger.warning(f"Failed to import module from {py_file}: {e}") @@ -80,7 +83,7 @@ def collect_components_from_directory(directory_path: str) -> dict: return components -def generate_node_json(node_name: str, component_class) -> dict: +def generate_node_json(node_name: str, component_class, file_path: str = "") -> dict: """Generate JSON structure for a single component.""" # Get description from docstring description = (component_class.__doc__ or "").strip() @@ -94,7 +97,7 @@ def generate_node_json(node_name: str, component_class) -> dict: display_name = f"Overridden {display_name}" description = f"This overrides the standard {node_name} node" else: - source = {"type": "custom", "data": f"custom://{node_name}"} + source = {"type": "custom", "data": f"custom://{file_path}/{node_name}"} # Build inputs structure inputs = {} @@ -150,8 +153,10 @@ def gen_json(directory_path: str): custom_nodes = [] stdlib_nodes = [] - for node_name, component_class in components.items(): - nodes[node_name] = generate_node_json(node_name, component_class) + for node_name, component_info in components.items(): + component_class = component_info["class"] + file_path = component_info["file_path"] + nodes[node_name] = generate_node_json(node_name, component_class, file_path) if is_stdlib_node(node_name): stdlib_nodes.append(node_name) @@ -178,30 +183,13 @@ def gen_json(directory_path: str): print(f"Stdlib overrides: {stdlib_nodes}") -def gen(path: str): - """Generate TypeScript files for a module.""" - print(f"Generating TypeScript files for module {path}") - module = py_path_to_module(path) - mod = importlib.import_module(module) - ts_file_path = path.replace(".py", ".flyde.ts") - typescript = 'import { CodeNode } from "@flyde/core";\n\n' - for name in mod.__dict__.keys(): - c = getattr(mod, name) - if name != "Component" and isinstance(c, type) and issubclass(c, Component): - typescript += c.to_ts(name) - - print(f"Writing TypeScript to {ts_file_path}") - with open(ts_file_path, "w") as f: - f.write(typescript) - - def main(): parser = argparse.ArgumentParser( description="""PyFlyde CLI that runs Flyde graphs and provides other useful functions. Examples: flyde.py path/to/MyFlow.flyde # Runs a flow - flyde.py gen path/to/module.py # Generates TS files for visual editor + flyde.py gen path/to/directory/ # Generates .flyde-nodes.json for directory """, formatter_class=argparse.RawDescriptionHelpFormatter, ) @@ -216,7 +204,7 @@ def main(): parser.add_argument( "path", type=str, - help='Path to a ".flyde" flow file to run, a Python ".py" module, or a directory to generate definitions for', + help='Path to a ".flyde" flow file to run, or a directory to generate .flyde-nodes.json for', ) args = parser.parse_args() @@ -237,10 +225,10 @@ def main(): # Add current folder to path when resolving modules relative to the current folder add_folder_to_path(".") - # Check if path is a directory or file + # Generate JSON for directory if os.path.isdir(args.path): gen_json(args.path) else: - gen(args.path) + raise ValueError(f"Path {args.path} is not a directory. Only directory generation is supported.") else: raise ValueError(f"Unknown command: {args.command}") diff --git a/flyde/cli.pyi b/flyde/cli.pyi index ea8d23b..925fdb8 100644 --- a/flyde/cli.pyi +++ b/flyde/cli.pyi @@ -12,10 +12,8 @@ def is_stdlib_node(node_name: str) -> bool: """Check if a node name matches a stdlib node.""" def collect_components_from_directory(directory_path: str) -> dict: """Collect all Component subclasses from .py files in a directory.""" -def generate_node_json(node_name: str, component_class) -> dict: +def generate_node_json(node_name: str, component_class, file_path: str = '') -> dict: """Generate JSON structure for a single component.""" def gen_json(directory_path: str): """Generate JSON file for all components in a directory.""" -def gen(path: str): - """Generate TypeScript files for a module.""" def main() -> None: ... diff --git a/flyde/flow.py b/flyde/flow.py index e097e4b..d4c10db 100644 --- a/flyde/flow.py +++ b/flyde/flow.py @@ -1,4 +1,5 @@ import importlib +import importlib.util import logging import os import sys @@ -41,8 +42,8 @@ def _preload_imports(self, base_path: str, imports: dict[str, list[str]]): # Save the blueprint YAML for the graph to be instantiated later self._graphs[node_id] = yml["node"] continue - # Translate typescript file path to python module - module = module.replace("/", ".").replace(".flyde.ts", "").replace("@", "") + # Convert module path format + module = module.replace("/", ".").replace("@", "") logger.debug(f"Importing module {module}") mod = importlib.import_module(module) for class_name in classes: @@ -65,11 +66,62 @@ def _load_component(self, name: str, path: str): if name in self._components: return - # Translate typescript file path to python module - path = path.replace("/", ".").replace(".flyde.ts", "").replace("@", "") - logger.debug(f"Importing module {path}") - mod = importlib.import_module(path) - self._components[name] = getattr(mod, name) + # Handle custom://path/to/mod.py/ClassName format + if path.startswith("custom://"): + custom_path = path[9:] # Remove "custom://" prefix + if "/" in custom_path and custom_path.count("/") >= 1: + # Split into module path and class name + parts = custom_path.rsplit("/", 1) + if len(parts) == 2: + module_path, class_name = parts + + # Resolve the module path relative to the flow file's directory + if module_path.endswith(".py"): + # It's a file path, resolve it relative to the flow file directory + absolute_module_path = os.path.join(self._base_path, module_path) + # Convert to module name for importing + spec = importlib.util.spec_from_file_location(class_name, absolute_module_path) + if spec and spec.loader: + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + self._components[name] = getattr(mod, class_name) + return + else: + # It's already a module path, convert file path to module path + module_path = module_path.replace("/", ".").replace(".py", "") + + # Add the flow file's directory to sys.path temporarily for relative imports + original_path = sys.path[:] + if self._base_path not in sys.path: + sys.path.insert(0, self._base_path) + + try: + logger.debug(f"Importing custom module {module_path}, class {class_name}") + mod = importlib.import_module(module_path) + self._components[name] = getattr(mod, class_name) + return + finally: + # Restore original sys.path + sys.path[:] = original_path + + # Handle @flyde/nodes package format for stdlib components + if path == "@flyde/nodes": + logger.debug(f"Loading stdlib component {name}") + from flyde.nodes import Conditional, GetAttribute, Http, InlineValue + + stdlib_components = { + "InlineValue": InlineValue, + "Conditional": Conditional, + "GetAttribute": GetAttribute, + "Http": Http, + } + if name in stdlib_components: + self._components[name] = stdlib_components[name] + return + + raise ValueError( + f"Invalid component source path: {path}. Only custom:// and @flyde/nodes formats are supported." + ) def create_graph(self, name: str, args: InstanceArgs): if name not in self._graphs: diff --git a/flyde/node.py b/flyde/node.py index 40b1754..403aacd 100644 --- a/flyde/node.py +++ b/flyde/node.py @@ -29,10 +29,12 @@ class InstanceSourceType(Enum): """InstanceSourceType is the source type of an instance. FILE: The instance is created from a file. - PACKAGE: The instance is created from a built in package.""" + PACKAGE: The instance is created from a built in package. + CUSTOM: The instance is created from a custom module with path format.""" FILE = "file" PACKAGE = "package" + CUSTOM = "custom" @dataclass @@ -318,41 +320,6 @@ def stop(self): logger.debug(f"Stopping {self._id}") self._stop.set() - @classmethod - def to_ts(cls, name: str = "") -> str: - """Convert the node to a TypeScript definition.""" - - name = cls.__name__ if name == "" else name # type: ignore - - inputs_str = "" - if hasattr(cls, "inputs") and len(cls.inputs) > 0: - inputs_str = ( - "\n" - + ",\n".join([f' {k}: {{ description: "{v.description}" }}' for k, v in cls.inputs.items()]) - + "\n" - ) - outputs_str = "" - if hasattr(cls, "outputs") and len(cls.outputs) > 0: - outputs_str = ( - "\n" - + ",\n".join([f' {k}: {{ description: "{v.description}" }}' for k, v in cls.outputs.items()]) - + "\n" - ) - - safe_doc = "" - if hasattr(cls, "__doc__") and cls.__doc__: - safe_doc = cls.__doc__.replace("\n", "\\n").replace("\r", "\\r").replace('"', '\\"') - - return ( - f"export const {name}: CodeNode = {{\n" - f' id: "{name}",\n' - f' description: "{safe_doc}",\n' - f" inputs: {{{inputs_str} }},\n" - f" outputs: {{{outputs_str} }},\n" - f" run: () => {{ return; }},\n" - f"}};\n\n" - ) - class Graph(Node): """A visual graph node that contains other nodes.""" diff --git a/flyde/node.pyi b/flyde/node.pyi index 9c791b8..2f1968b 100644 --- a/flyde/node.pyi +++ b/flyde/node.pyi @@ -23,9 +23,11 @@ class InstanceSourceType(Enum): """InstanceSourceType is the source type of an instance. FILE: The instance is created from a file. - PACKAGE: The instance is created from a built in package.""" + PACKAGE: The instance is created from a built in package. + CUSTOM: The instance is created from a custom module with path format.""" FILE = 'file' PACKAGE = 'package' + CUSTOM = 'custom' @dataclass class InstanceSource: @@ -99,9 +101,6 @@ class Component(Node): def run(self) -> None: ... def stop(self) -> None: """Stop the component execution.""" - @classmethod - def to_ts(cls, name: str = '') -> str: - """Convert the node to a TypeScript definition.""" class Graph(Node): """A visual graph node that contains other nodes.""" diff --git a/tests/.flyde-nodes.json b/tests/.flyde-nodes.json index 5ca819e..15678b0 100644 --- a/tests/.flyde-nodes.json +++ b/tests/.flyde-nodes.json @@ -7,7 +7,7 @@ "description": "A component that capitalizes the input.", "source": { "type": "custom", - "data": "custom://Capitalize" + "data": "custom://components.py/Capitalize" }, "editorNode": { "id": "Capitalize", @@ -38,7 +38,7 @@ "description": "A simple component that echoes the input.", "source": { "type": "custom", - "data": "custom://Echo" + "data": "custom://components.py/Echo" }, "editorNode": { "id": "Echo", @@ -69,7 +69,7 @@ "description": "Formats the input value with a given format string and sends it to out.", "source": { "type": "custom", - "data": "custom://Format" + "data": "custom://components.py/Format" }, "editorNode": { "id": "Format", @@ -103,7 +103,7 @@ "description": "A component that has both inputs and outputs and a sticky input.", "source": { "type": "custom", - "data": "custom://RepeatWordNTimes" + "data": "custom://components.py/RepeatWordNTimes" }, "editorNode": { "id": "RepeatWordNTimes", @@ -137,7 +137,7 @@ "description": "A custom external node named Bob", "source": { "type": "custom", - "data": "custom://CustomBob" + "data": "custom://test_cli.py/CustomBob" }, "editorNode": { "id": "CustomBob", @@ -193,7 +193,7 @@ "description": "A test component for unit testing.", "source": { "type": "custom", - "data": "custom://TestCustomComponent" + "data": "custom://test_cli.py/TestCustomComponent" }, "editorNode": { "id": "TestCustomComponent", @@ -227,7 +227,7 @@ "description": "A component with only sticky inputs to test single execution.", "source": { "type": "custom", - "data": "custom://AllStickyInputsComponent" + "data": "custom://test_node.py/AllStickyInputsComponent" }, "editorNode": { "id": "AllStickyInputsComponent", @@ -261,7 +261,7 @@ "description": "A component that has a custom run and shutdown handlers.", "source": { "type": "custom", - "data": "custom://CustomRunComponent" + "data": "custom://test_node.py/CustomRunComponent" }, "editorNode": { "id": "CustomRunComponent", @@ -292,7 +292,7 @@ "description": "A component that tries to send a message without a corresponding output.", "source": { "type": "custom", - "data": "custom://InvalidSendProcess" + "data": "custom://test_node.py/InvalidSendProcess" }, "editorNode": { "id": "InvalidSendProcess", @@ -319,7 +319,7 @@ "description": "A component to test no inputs, outputs, and no process method.", "source": { "type": "custom", - "data": "custom://NoProcessComponent" + "data": "custom://test_node.py/NoProcessComponent" }, "editorNode": { "id": "NoProcessComponent", @@ -342,7 +342,7 @@ "description": "A component that only has inputs.", "source": { "type": "custom", - "data": "custom://SinkComponent" + "data": "custom://test_node.py/SinkComponent" }, "editorNode": { "id": "SinkComponent", @@ -372,7 +372,7 @@ "description": "A component that only has outputs.", "source": { "type": "custom", - "data": "custom://SourceComponent" + "data": "custom://test_node.py/SourceComponent" }, "editorNode": { "id": "SourceComponent", diff --git a/tests/Repeat3Times.flyde b/tests/Repeat3Times.flyde index 6977be4..0a1dea9 100644 --- a/tests/Repeat3Times.flyde +++ b/tests/Repeat3Times.flyde @@ -18,8 +18,8 @@ node: value: "{{times}}" type: code source: - type: file - data: components.flyde.ts + type: custom + data: custom://components.py/RepeatWordNTimes - pos: x: -422.4172058105469 y: 126.0786114501953 @@ -46,8 +46,8 @@ node: value: "{{inp}}" type: code source: - type: file - data: components.flyde.ts + type: custom + data: custom://components.py/Capitalize connections: - from: insId: inl-ytsuyrje4syeb4qduymsfkl2 diff --git a/tests/TestFanIn.flyde b/tests/TestFanIn.flyde index 35dd6d6..ddf9114 100644 --- a/tests/TestFanIn.flyde +++ b/tests/TestFanIn.flyde @@ -16,8 +16,8 @@ node: value: "{{format}}" type: code source: - type: file - data: components.flyde.ts + type: custom + data: custom://components.py/Format - pos: x: -276.1663818359375 y: -58.34027099609375 @@ -30,8 +30,8 @@ node: value: "{{inp}}" type: code source: - type: file - data: components.flyde.ts + type: custom + data: custom://components.py/Capitalize - pos: x: -5.659755859374968 y: 54.9669189453125 @@ -44,8 +44,8 @@ node: value: "{{inp}}" type: code source: - type: file - data: components.flyde.ts + type: custom + data: custom://components.py/Echo - pos: x: -594.1567211914063 y: 120.8353289794922 diff --git a/tests/TestFanInGraph.flyde b/tests/TestFanInGraph.flyde index c048c85..3a0bcad 100644 --- a/tests/TestFanInGraph.flyde +++ b/tests/TestFanInGraph.flyde @@ -16,8 +16,8 @@ node: value: "{{format}}" type: code source: - type: file - data: components.flyde.ts + type: custom + data: custom://components.py/Format - pos: x: -58.16827392578125 y: 63.02734375 @@ -30,8 +30,8 @@ node: value: "{{inp}}" type: code source: - type: file - data: components.flyde.ts + type: custom + data: custom://components.py/Capitalize - pos: x: -601.1567211914063 y: 121.8353289794922 diff --git a/tests/TestInOutFlow.flyde b/tests/TestInOutFlow.flyde index 3b223e3..d3abab8 100644 --- a/tests/TestInOutFlow.flyde +++ b/tests/TestInOutFlow.flyde @@ -13,8 +13,8 @@ node: value: "{{inp}}" type: code source: - type: file - data: components.flyde.ts + type: custom + data: custom://components.py/Echo - pos: x: -17.077154541015602 y: -87.20937501999856 @@ -49,8 +49,8 @@ node: value: "{{format}}" type: code source: - type: file - data: components.flyde.ts + type: custom + data: custom://components.py/Format - pos: x: -32.78351928710936 y: 33.626827372579555 diff --git a/tests/TestIsolatedFlow.flyde b/tests/TestIsolatedFlow.flyde index 0789c69..9d7fdee 100644 --- a/tests/TestIsolatedFlow.flyde +++ b/tests/TestIsolatedFlow.flyde @@ -27,8 +27,8 @@ node: value: "{{inp}}" type: code source: - type: file - data: components.flyde.ts + type: custom + data: custom://components.py/Echo connections: - from: insId: qxavnllf1foh9ivxvtz2hkad diff --git a/tests/TestNestedFlow.flyde b/tests/TestNestedFlow.flyde index 5eb664e..921865d 100644 --- a/tests/TestNestedFlow.flyde +++ b/tests/TestNestedFlow.flyde @@ -13,8 +13,8 @@ node: value: "{{inp}}" type: code source: - type: file - data: components.flyde.ts + type: custom + data: custom://components.py/Echo - pos: x: -195.52671508789064 y: 269.8743734741211 @@ -32,8 +32,8 @@ node: value: "{{times}}" type: code source: - type: file - data: components.flyde.ts + type: custom + data: custom://components.py/RepeatWordNTimes - pos: x: -674.5008868408204 y: 195.91538436889653 diff --git a/tests/components.flyde.ts b/tests/components.flyde.ts deleted file mode 100644 index 1b38c90..0000000 --- a/tests/components.flyde.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { CodeNode } from "@flyde/core"; - -export const Format: CodeNode = { - id: "Format", - description: "Formats the input value with a given format string and sends it to out.", - inputs: { - inp: { description: "The input" }, - format: { description: "The format string" } - }, - outputs: { - out: { description: "The formatted output" } - }, - run: () => { return; }, -}; - -export const Echo: CodeNode = { - id: "Echo", - description: "A simple component that echoes the input.", - inputs: { - inp: { description: "The input" } - }, - outputs: { - out: { description: "The output" } - }, - run: () => { return; }, -}; - -export const Capitalize: CodeNode = { - id: "Capitalize", - description: "A component that capitalizes the input.", - inputs: { - inp: { description: "The input" } - }, - outputs: { - out: { description: "The output" } - }, - run: () => { return; }, -}; - -export const RepeatWordNTimes: CodeNode = { - id: "RepeatWordNTimes", - description: "A component that has both inputs and outputs and a sticky input.", - inputs: { - word: { description: "The input" }, - times: { description: "The number of times to repeat the input" } - }, - outputs: { - out: { description: "The output" } - }, - run: () => { return; }, -}; - diff --git a/tests/test_cli.py b/tests/test_cli.py index ea6d9ed..bca9fbe 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -92,7 +92,7 @@ def process(self): class TestGenerateNodeJson(unittest.TestCase): def test_generate_custom_node_json(self): - result = generate_node_json("CustomBob", CustomBob) + result = generate_node_json("CustomBob", CustomBob, "test_components.py") expected = { "id": "CustomBob", @@ -100,7 +100,7 @@ def test_generate_custom_node_json(self): "displayName": "Custom Bob", "description": "A custom external node named Bob", "icon": "fa-solid fa-user", - "source": {"type": "custom", "data": "custom://CustomBob"}, + "source": {"type": "custom", "data": "custom://test_components.py/CustomBob"}, "editorNode": { "id": "CustomBob", "displayName": "Custom Bob", @@ -116,7 +116,7 @@ def test_generate_custom_node_json(self): self.assertEqual(result, expected) def test_generate_stdlib_override_json(self): - result = generate_node_json("InlineValue", InlineValue) + result = generate_node_json("InlineValue", InlineValue, "test_components.py") expected = { "id": "InlineValue", @@ -142,7 +142,7 @@ class NodeWithoutDoc(Component): inputs = {"inp": Input(description="Input")} outputs = {"out": Output(description="Output")} - result = generate_node_json("NodeWithoutDoc", NodeWithoutDoc) + result = generate_node_json("NodeWithoutDoc", NodeWithoutDoc, "test.py") self.assertEqual(result["description"], "") self.assertEqual(result["editorNode"]["description"], "") @@ -151,7 +151,7 @@ def test_generate_node_with_no_inputs_outputs(self): class EmptyNode(Component): """An empty node""" - result = generate_node_json("EmptyNode", EmptyNode) + result = generate_node_json("EmptyNode", EmptyNode, "test.py") self.assertEqual(result["editorNode"]["inputs"], {}) self.assertEqual(result["editorNode"]["outputs"], {}) @@ -217,8 +217,9 @@ class AnotherNode(Component): self.assertIn("AnotherNode", components) # Check that components are actually Component subclasses - for name, cls in components.items(): - self.assertTrue(issubclass(cls, Component)) + for name, component_info in components.items(): + self.assertTrue(issubclass(component_info["class"], Component)) + self.assertIn("file_path", component_info) def test_collect_components_invalid_syntax(self): # Create a file with invalid Python syntax @@ -304,7 +305,7 @@ class InlineValue(Component): self.assertEqual(bob["id"], "CustomBob") self.assertEqual(bob["displayName"], "Custom Bob") self.assertEqual(bob["source"]["type"], "custom") - self.assertEqual(bob["source"]["data"], "custom://CustomBob") + self.assertEqual(bob["source"]["data"], "custom://components.py/CustomBob") self.assertIn("icon", bob) # Check InlineValue (stdlib override) @@ -326,6 +327,11 @@ class InlineValue(Component): self.assertCountEqual(custom_group["nodeIds"], ["CustomBob", "CustomAlice"]) self.assertCountEqual(stdlib_group["nodeIds"], ["InlineValue"]) + # Check CustomAlice has correct path format + alice = nodes["CustomAlice"] + self.assertEqual(alice["source"]["type"], "custom") + self.assertEqual(alice["source"]["data"], "custom://components.py/CustomAlice") + def test_gen_json_empty_directory(self): # Test with directory containing no components empty_py = """ @@ -380,6 +386,12 @@ class CustomNode2(Component): self.assertEqual(groups[0]["title"], "Custom Runtime Nodes") self.assertCountEqual(groups[0]["nodeIds"], ["CustomNode1", "CustomNode2"]) + # Check that custom nodes have correct path format + for node_name in ["CustomNode1", "CustomNode2"]: + node = data["nodes"][node_name] + self.assertEqual(node["source"]["type"], "custom") + self.assertEqual(node["source"]["data"], f"custom://components.py/{node_name}") + # Ensure no stdlib overrides group exists for group in groups: self.assertNotEqual(group["title"], "Overridden Stdlib") diff --git a/tests/test_flow.py b/tests/test_flow.py index 0cddcca..086994b 100644 --- a/tests/test_flow.py +++ b/tests/test_flow.py @@ -104,6 +104,74 @@ def test_with_component(self): flow.stopped.wait() self.assertTrue(flow.stopped.is_set()) + +class TestCustomLoadFlow(unittest.TestCase): + def test_custom_component_loading(self): + """Test loading components using custom:// source format.""" + test_case = { + "inputs": ["hello", "world", EOF], + "outputs": ["HELLO", "WORLD", EOF], + } + flow = Flow.from_file("tests/TestCustomLoad.flyde") + + in_q = flow.node.inputs["input"].queue + out_q = Queue() + flow.node.outputs["output"].connect(out_q) + + flow.run() + + for inp, expected_out in zip(test_case["inputs"], test_case["outputs"]): + in_q.put(inp) + if expected_out == EOF: + actual_out = out_q.get() + self.assertEqual(EOF, actual_out) + else: + actual_out = out_q.get() + self.assertEqual(expected_out, actual_out) + + flow.stopped.wait() + self.assertTrue(flow.stopped.is_set()) + + def test_custom_component_loading_edge_cases(self): + """Test edge cases in custom component loading.""" + from flyde.flow import Flow + from flyde.node import InstanceArgs, InstanceSource, InstanceSourceType + + flow = Flow({}) + + # Test malformed custom path (no class name) + args = InstanceArgs( + id="test", + display_name="Test", + stopped=None, + config={}, + source=InstanceSource(type=InstanceSourceType.CUSTOM, data="custom://tests/components.py/"), + ) + with self.assertRaises(Exception): + flow.create_component("Echo", args) + + # Test invalid module path + args = InstanceArgs( + id="test", + display_name="Test", + stopped=None, + config={}, + source=InstanceSource(type=InstanceSourceType.CUSTOM, data="custom://tests/nonexistent.py/Echo"), + ) + with self.assertRaises(Exception): + flow.create_component("Echo", args) + + # Test valid custom path + args = InstanceArgs( + id="test", + display_name="Test", + stopped=None, + config={}, + source=InstanceSource(type=InstanceSourceType.CUSTOM, data="custom://tests/components.py/Echo"), + ) + component = flow.create_component("Echo", args) + self.assertIsNotNone(component) + def test_with_graph(self): test_case = { "inputs": ["John", EOF], diff --git a/tests/test_node.py b/tests/test_node.py index 4750e9f..0342c7b 100644 --- a/tests/test_node.py +++ b/tests/test_node.py @@ -117,28 +117,6 @@ def test_static_input(self): self.assertEqual(out_q.get(), EOF) self.assertEqual(in_q.qsize(), 0) - def test_to_ts(self): - self.maxDiff = None - - def expected_typescript(name): - return """export const {NAME}: CodeNode = { - id: "{NAME}", - description: "A component that has both inputs and outputs and a sticky input.", - inputs: { - word: { description: "The input" }, - times: { description: "The number of times to repeat the input" } - }, - outputs: { - out: { description: "The output" } - }, - run: () => { return; }, -}; - -""".replace("{NAME}", name) - - self.assertEqual(RepeatWordNTimes.to_ts(), expected_typescript("RepeatWordNTimes")) - self.assertEqual(RepeatWordNTimes.to_ts("RepeatWord"), expected_typescript("RepeatWord")) - def test_from_yaml(self): yaml = { "id": "repeat", From 40447822d350f425c48c695935f27e9ca003e26d Mon Sep 17 00:00:00 2001 From: Vladimir Sibirov Date: Sun, 13 Jul 2025 21:48:06 +0200 Subject: [PATCH 11/18] Add missing files --- examples/.flyde-nodes.json | 274 +++++++++++++++++++++++++++++++++++++ tests/TestCustomLoad.flyde | 66 +++++++++ 2 files changed, 340 insertions(+) create mode 100644 examples/.flyde-nodes.json create mode 100644 tests/TestCustomLoad.flyde diff --git a/examples/.flyde-nodes.json b/examples/.flyde-nodes.json new file mode 100644 index 0000000..edf575c --- /dev/null +++ b/examples/.flyde-nodes.json @@ -0,0 +1,274 @@ +{ + "nodes": { + "LoadDataset": { + "id": "LoadDataset", + "type": "code", + "displayName": "Load Dataset", + "description": "Loads a dataset from a file into a DataFrame.", + "source": { + "type": "custom", + "data": "custom://mylib/dataframe.py/LoadDataset" + }, + "editorNode": { + "id": "LoadDataset", + "displayName": "Load Dataset", + "description": "Loads a dataset from a file into a DataFrame.", + "inputs": { + "file_path": { + "description": "The path to the file containing the dataset" + } + }, + "outputs": { + "dataframe": { + "description": "The loaded dataframe" + } + }, + "editorConfig": { + "type": "structured" + }, + "icon": "fa-solid fa-user" + }, + "config": {}, + "icon": "fa-solid fa-user" + }, + "Scale": { + "id": "Scale", + "type": "code", + "displayName": "Scale", + "description": "Scales the features of a dataframe with a scikit-learn StandardScaler.", + "source": { + "type": "custom", + "data": "custom://mylib/dataframe.py/Scale" + }, + "editorNode": { + "id": "Scale", + "displayName": "Scale", + "description": "Scales the features of a dataframe with a scikit-learn StandardScaler.", + "inputs": { + "dataframe": { + "description": "The dataframe to scale" + } + }, + "outputs": { + "scaled_dataframe": { + "description": "The scaled dataframe" + } + }, + "editorConfig": { + "type": "structured" + }, + "icon": "fa-solid fa-user" + }, + "config": {}, + "icon": "fa-solid fa-user" + }, + "KMeansCluster": { + "id": "KMeansCluster", + "type": "code", + "displayName": "KMeans Cluster", + "description": "Clusters the dataframe using K-means clustering.", + "source": { + "type": "custom", + "data": "custom://mylib/kmeans.py/KMeansCluster" + }, + "editorNode": { + "id": "KMeansCluster", + "displayName": "KMeans Cluster", + "description": "Clusters the dataframe using K-means clustering.", + "inputs": { + "scaled_dataframe": { + "description": "The scaled dataframe to cluster" + }, + "n_clusters": { + "description": "The number of clusters" + } + }, + "outputs": { + "kmeans_result": { + "description": "K-means clustering result" + } + }, + "editorConfig": { + "type": "structured" + }, + "icon": "fa-solid fa-user" + }, + "config": {}, + "icon": "fa-solid fa-user" + }, + "KMeansNClusters": { + "id": "KMeansNClusters", + "type": "code", + "displayName": "KMeans NClusters", + "description": "Finds the optimal number of clusters for K-means clustering using silhouette method.", + "source": { + "type": "custom", + "data": "custom://mylib/kmeans.py/KMeansNClusters" + }, + "editorNode": { + "id": "KMeansNClusters", + "displayName": "KMeans NClusters", + "description": "Finds the optimal number of clusters for K-means clustering using silhouette method.", + "inputs": { + "scaled_dataframe": { + "description": "The scaled dataframe to cluster" + }, + "max_clusters": { + "description": "The maximum number of clusters to consider" + } + }, + "outputs": { + "n_clusters": { + "description": "The optimal number of clusters" + } + }, + "editorConfig": { + "type": "structured" + }, + "icon": "fa-solid fa-user" + }, + "config": {}, + "icon": "fa-solid fa-user" + }, + "PCA2": { + "id": "PCA2", + "type": "code", + "displayName": "PCA2", + "description": "Performs PCA on a dataframe and returns the first two principal components.", + "source": { + "type": "custom", + "data": "custom://mylib/kmeans.py/PCA2" + }, + "editorNode": { + "id": "PCA2", + "displayName": "PCA2", + "description": "Performs PCA on a dataframe and returns the first two principal components.", + "inputs": { + "scaled_dataframe": { + "description": "The scaled dataframe to reduce" + } + }, + "outputs": { + "pca_components": { + "description": "The first two principal components" + } + }, + "editorConfig": { + "type": "structured" + }, + "icon": "fa-solid fa-user" + }, + "config": {}, + "icon": "fa-solid fa-user" + }, + "Visualize": { + "id": "Visualize", + "type": "code", + "displayName": "Visualize", + "description": "Visualizes the clustered dataframe.", + "source": { + "type": "custom", + "data": "custom://mylib/kmeans.py/Visualize" + }, + "editorNode": { + "id": "Visualize", + "displayName": "Visualize", + "description": "Visualizes the clustered dataframe.", + "inputs": { + "pca_components": { + "description": "The first two principal components" + }, + "pca_centroids": { + "description": "The centroids in PCA space" + }, + "kmeans_result": { + "description": "K-means clustering result" + } + }, + "outputs": {}, + "editorConfig": { + "type": "structured" + }, + "icon": "fa-solid fa-user" + }, + "config": {}, + "icon": "fa-solid fa-user" + }, + "Concat": { + "id": "Concat", + "type": "code", + "displayName": "Concat", + "description": "Concatenates two strings.", + "source": { + "type": "custom", + "data": "custom://mylib/components.py/Concat" + }, + "editorNode": { + "id": "Concat", + "displayName": "Concat", + "description": "Concatenates two strings.", + "inputs": { + "a": { + "description": "The first string" + }, + "b": { + "description": "The second string" + } + }, + "outputs": { + "out": { + "description": "The concatenated string" + } + }, + "editorConfig": { + "type": "structured" + }, + "icon": "fa-solid fa-user" + }, + "config": {}, + "icon": "fa-solid fa-user" + }, + "Print": { + "id": "Print", + "type": "code", + "displayName": "Print", + "description": "Prints the input message to the console.", + "source": { + "type": "custom", + "data": "custom://mylib/components.py/Print" + }, + "editorNode": { + "id": "Print", + "displayName": "Print", + "description": "Prints the input message to the console.", + "inputs": { + "msg": { + "description": "The message to print" + } + }, + "outputs": {}, + "editorConfig": { + "type": "structured" + }, + "icon": "fa-solid fa-user" + }, + "config": {}, + "icon": "fa-solid fa-user" + } + }, + "groups": [ + { + "title": "Custom Runtime Nodes", + "nodeIds": [ + "LoadDataset", + "Scale", + "KMeansCluster", + "KMeansNClusters", + "PCA2", + "Visualize", + "Concat", + "Print" + ] + } + ] +} \ No newline at end of file diff --git a/tests/TestCustomLoad.flyde b/tests/TestCustomLoad.flyde new file mode 100644 index 0000000..3592879 --- /dev/null +++ b/tests/TestCustomLoad.flyde @@ -0,0 +1,66 @@ +imports: {} +node: + instances: + - pos: + x: -200 + y: 50 + id: custom-echo-test + inputConfig: {} + nodeId: Echo + config: + inp: + type: dynamic + value: "{{inp}}" + type: code + source: + type: custom + data: custom://components.py/Echo + - pos: + x: 50 + y: 50 + id: custom-capitalize-test + inputConfig: {} + nodeId: Capitalize + config: + inp: + type: dynamic + value: "{{inp}}" + type: code + source: + type: custom + data: custom://components.py/Capitalize + connections: + - from: + insId: __this + pinId: input + to: + insId: custom-echo-test + pinId: inp + - from: + insId: custom-echo-test + pinId: out + to: + insId: custom-capitalize-test + pinId: inp + - from: + insId: custom-capitalize-test + pinId: out + to: + insId: __this + pinId: output + id: TestCustomLoad + inputs: + input: + mode: required + outputs: + output: + delayed: false + inputsPosition: + input: + x: -350 + y: 50 + outputsPosition: + output: + x: 200 + y: 50 + description: Test flow that loads custom components using custom:// source format From 9e2a1c125ad02dd5c96239a0af2dc42ff7e7d060 Mon Sep 17 00:00:00 2001 From: Vladimir Sibirov Date: Sun, 20 Jul 2025 19:52:34 +0200 Subject: [PATCH 12/18] Experimenting with stdlib overrids --- examples/.flyde-nodes.json | 200 ++++++++++++++++++++++++++++++------ flyde/cli.py | 57 ++++++----- flyde/flow.py | 14 ++- flyde/nodes.py | 75 ++++++++++---- tests/.flyde-nodes.json | 202 ++++++++++++++++++++++++++++--------- tests/test_cli.py | 116 ++++++++++----------- tests/test_flow.py | 13 ++- 7 files changed, 478 insertions(+), 199 deletions(-) diff --git a/examples/.flyde-nodes.json b/examples/.flyde-nodes.json index edf575c..bdf83ce 100644 --- a/examples/.flyde-nodes.json +++ b/examples/.flyde-nodes.json @@ -5,6 +5,7 @@ "type": "code", "displayName": "Load Dataset", "description": "Loads a dataset from a file into a DataFrame.", + "icon": "fa-solid fa-user", "source": { "type": "custom", "data": "custom://mylib/dataframe.py/LoadDataset" @@ -25,17 +26,16 @@ }, "editorConfig": { "type": "structured" - }, - "icon": "fa-solid fa-user" + } }, - "config": {}, - "icon": "fa-solid fa-user" + "config": {} }, "Scale": { "id": "Scale", "type": "code", "displayName": "Scale", "description": "Scales the features of a dataframe with a scikit-learn StandardScaler.", + "icon": "fa-solid fa-user", "source": { "type": "custom", "data": "custom://mylib/dataframe.py/Scale" @@ -56,17 +56,16 @@ }, "editorConfig": { "type": "structured" - }, - "icon": "fa-solid fa-user" + } }, - "config": {}, - "icon": "fa-solid fa-user" + "config": {} }, "KMeansCluster": { "id": "KMeansCluster", "type": "code", "displayName": "KMeans Cluster", "description": "Clusters the dataframe using K-means clustering.", + "icon": "fa-solid fa-user", "source": { "type": "custom", "data": "custom://mylib/kmeans.py/KMeansCluster" @@ -90,17 +89,16 @@ }, "editorConfig": { "type": "structured" - }, - "icon": "fa-solid fa-user" + } }, - "config": {}, - "icon": "fa-solid fa-user" + "config": {} }, "KMeansNClusters": { "id": "KMeansNClusters", "type": "code", "displayName": "KMeans NClusters", "description": "Finds the optimal number of clusters for K-means clustering using silhouette method.", + "icon": "fa-solid fa-user", "source": { "type": "custom", "data": "custom://mylib/kmeans.py/KMeansNClusters" @@ -124,17 +122,16 @@ }, "editorConfig": { "type": "structured" - }, - "icon": "fa-solid fa-user" + } }, - "config": {}, - "icon": "fa-solid fa-user" + "config": {} }, "PCA2": { "id": "PCA2", "type": "code", "displayName": "PCA2", "description": "Performs PCA on a dataframe and returns the first two principal components.", + "icon": "fa-solid fa-user", "source": { "type": "custom", "data": "custom://mylib/kmeans.py/PCA2" @@ -155,17 +152,16 @@ }, "editorConfig": { "type": "structured" - }, - "icon": "fa-solid fa-user" + } }, - "config": {}, - "icon": "fa-solid fa-user" + "config": {} }, "Visualize": { "id": "Visualize", "type": "code", "displayName": "Visualize", "description": "Visualizes the clustered dataframe.", + "icon": "fa-solid fa-user", "source": { "type": "custom", "data": "custom://mylib/kmeans.py/Visualize" @@ -188,17 +184,16 @@ "outputs": {}, "editorConfig": { "type": "structured" - }, - "icon": "fa-solid fa-user" + } }, - "config": {}, - "icon": "fa-solid fa-user" + "config": {} }, "Concat": { "id": "Concat", "type": "code", "displayName": "Concat", "description": "Concatenates two strings.", + "icon": "fa-solid fa-user", "source": { "type": "custom", "data": "custom://mylib/components.py/Concat" @@ -222,17 +217,16 @@ }, "editorConfig": { "type": "structured" - }, - "icon": "fa-solid fa-user" + } }, - "config": {}, - "icon": "fa-solid fa-user" + "config": {} }, "Print": { "id": "Print", "type": "code", "displayName": "Print", "description": "Prints the input message to the console.", + "icon": "fa-solid fa-user", "source": { "type": "custom", "data": "custom://mylib/components.py/Print" @@ -249,11 +243,146 @@ "outputs": {}, "editorConfig": { "type": "structured" + } + }, + "config": {} + }, + "Conditional": { + "id": "Conditional", + "type": "code", + "displayName": "Conditional", + "description": "Conditional component evaluates a condition against the input and sends the result to output.", + "icon": "circle-question", + "source": { + "type": "package", + "data": "@flyde/nodes" + }, + "editorNode": { + "id": "Conditional", + "displayName": "Conditional", + "description": "Conditional component evaluates a condition against the input and sends the result to output.", + "inputs": { + "leftOperand": { + "description": "Left operand of the condition" + }, + "rightOperand": { + "description": "Right operand of the condition" + } + }, + "outputs": { + "true": { + "description": "Output when the condition is true" + }, + "false": { + "description": "Output when the condition is false" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "GetAttribute": { + "id": "GetAttribute", + "type": "code", + "displayName": "Get Attribute", + "description": "Get an attribute from an object or dictionary.", + "icon": "fa-magnifying-glass", + "source": { + "type": "package", + "data": "@flyde/nodes" + }, + "editorNode": { + "id": "GetAttribute", + "displayName": "Get Attribute", + "description": "Get an attribute from an object or dictionary.", + "inputs": { + "object": { + "description": "The object or dictionary" + }, + "key": { + "description": "The attribute name" + } + }, + "outputs": { + "value": { + "description": "The attribute value" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "Http": { + "id": "Http", + "type": "code", + "displayName": "Http", + "description": "Http component makes HTTP requests with urllib.", + "icon": "globe", + "source": { + "type": "package", + "data": "@flyde/nodes" + }, + "editorNode": { + "id": "Http", + "displayName": "Http", + "description": "Http component makes HTTP requests with urllib.", + "inputs": { + "url": { + "description": "URL to request" + }, + "method": { + "description": "HTTP method" + }, + "headers": { + "description": "HTTP headers" + }, + "params": { + "description": "URL parameters" + }, + "data": { + "description": "Request body" + } + }, + "outputs": { + "data": { + "description": "Response data" + } }, - "icon": "fa-solid fa-user" + "editorConfig": { + "type": "structured" + } }, - "config": {}, - "icon": "fa-solid fa-user" + "config": {} + }, + "InlineValue": { + "id": "InlineValue", + "type": "code", + "displayName": "Inline Value", + "description": "InlineValue sends a constant value to output.", + "icon": "pencil", + "source": { + "type": "package", + "data": "@flyde/nodes" + }, + "editorNode": { + "id": "InlineValue", + "displayName": "Inline Value", + "description": "InlineValue sends a constant value to output.", + "inputs": {}, + "outputs": { + "value": { + "description": "The constant value" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} } }, "groups": [ @@ -269,6 +398,15 @@ "Concat", "Print" ] + }, + { + "title": "Overridden Stdlib", + "nodeIds": [ + "Conditional", + "GetAttribute", + "Http", + "InlineValue" + ] } ] } \ No newline at end of file diff --git a/flyde/cli.py b/flyde/cli.py index d401e63..133c8f3 100644 --- a/flyde/cli.py +++ b/flyde/cli.py @@ -7,6 +7,7 @@ import logging import os import pprint +import re import sys from flyde.flow import Flow, add_folder_to_path @@ -23,8 +24,6 @@ def py_path_to_module(py_path: str) -> str: def convert_class_name_to_display_name(class_name: str) -> str: """Convert a class name like 'MyCustomNode' to 'My Custom Node'.""" - # Insert space before uppercase letters that follow lowercase letters - import re return re.sub(r"(?<=[a-z])(?=[A-Z])", " ", class_name) @@ -85,31 +84,31 @@ def collect_components_from_directory(directory_path: str) -> dict: def generate_node_json(node_name: str, component_class, file_path: str = "") -> dict: """Generate JSON structure for a single component.""" - # Get description from docstring + # Get node metadata description = (component_class.__doc__ or "").strip() - - # Generate display name display_name = convert_class_name_to_display_name(node_name) - - # Determine source + # Use package source for stdlib nodes, custom source for others if is_stdlib_node(node_name): source = {"type": "package", "data": "@flyde/nodes"} - display_name = f"Overridden {display_name}" - description = f"This overrides the standard {node_name} node" else: source = {"type": "custom", "data": f"custom://{file_path}/{node_name}"} + icon = getattr(component_class, "icon", "fa-solid fa-user") # Build inputs structure inputs = {} if hasattr(component_class, "inputs") and component_class.inputs: for input_name, input_obj in component_class.inputs.items(): - inputs[input_name] = {"description": input_obj.description or f"{input_name} input"} + inputs[input_name] = { + "description": input_obj.description or f"{input_name} input" + } # Build outputs structure outputs = {} if hasattr(component_class, "outputs") and component_class.outputs: for output_name, output_obj in component_class.outputs.items(): - outputs[output_name] = {"description": output_obj.description or f"{output_name} output"} + outputs[output_name] = { + "description": output_obj.description or f"{output_name} output" + } # Build the node structure node_data = { @@ -117,6 +116,7 @@ def generate_node_json(node_name: str, component_class, file_path: str = "") -> "type": "code", "displayName": display_name, "description": description, + "icon": icon, "source": source, "editorNode": { "id": node_name, @@ -129,11 +129,6 @@ def generate_node_json(node_name: str, component_class, file_path: str = "") -> "config": {}, } - # Add icon for custom nodes - if not is_stdlib_node(node_name): - node_data["icon"] = "fa-solid fa-user" - node_data["editorNode"]["icon"] = "fa-solid fa-user" - return node_data @@ -144,8 +139,13 @@ def gen_json(directory_path: str): # Collect all components components = collect_components_from_directory(directory_path) - if not components: - print(f"No Component subclasses found in directory {directory_path}") + # Always include stdlib nodes from flyde/nodes.py + stdlib_dir = os.path.join(os.path.dirname(__file__), "nodes.py") + stdlib_components = collect_components_from_directory(os.path.dirname(stdlib_dir)) + stdlib_node_names = [name for name in stdlib_components if is_stdlib_node(name)] + + if not components and not stdlib_node_names: + print(f"No Component subclasses found in directory {directory_path} or stdlib") return # Build nodes structure @@ -153,15 +153,20 @@ def gen_json(directory_path: str): custom_nodes = [] stdlib_nodes = [] + # Add user components for node_name, component_info in components.items(): component_class = component_info["class"] file_path = component_info["file_path"] nodes[node_name] = generate_node_json(node_name, component_class, file_path) + custom_nodes.append(node_name) - if is_stdlib_node(node_name): - stdlib_nodes.append(node_name) - else: - custom_nodes.append(node_name) + # Add stdlib nodes (from flyde/nodes.py) as custom nodes, but group as stdlib overrides + for node_name in stdlib_node_names: + if node_name not in nodes: + component_class = stdlib_components[node_name]["class"] + file_path = stdlib_components[node_name]["file_path"] + nodes[node_name] = generate_node_json(node_name, component_class, file_path) + stdlib_nodes.append(node_name) # Build groups groups = [] @@ -178,9 +183,9 @@ def gen_json(directory_path: str): with open(output_file, "w") as f: json.dump(json_data, f, indent=2) - print(f"Generated {output_file} with {len(components)} components") + print(f"Generated {output_file} with {len(nodes)} components") print(f"Custom nodes: {custom_nodes}") - print(f"Stdlib overrides: {stdlib_nodes}") + print(f"Stdlib nodes: {stdlib_nodes}") def main(): @@ -229,6 +234,8 @@ def main(): if os.path.isdir(args.path): gen_json(args.path) else: - raise ValueError(f"Path {args.path} is not a directory. Only directory generation is supported.") + raise ValueError( + f"Path {args.path} is not a directory. Only directory generation is supported." + ) else: raise ValueError(f"Unknown command: {args.command}") diff --git a/flyde/flow.py b/flyde/flow.py index d4c10db..d677069 100644 --- a/flyde/flow.py +++ b/flyde/flow.py @@ -78,9 +78,13 @@ def _load_component(self, name: str, path: str): # Resolve the module path relative to the flow file's directory if module_path.endswith(".py"): # It's a file path, resolve it relative to the flow file directory - absolute_module_path = os.path.join(self._base_path, module_path) + absolute_module_path = os.path.join( + self._base_path, module_path + ) # Convert to module name for importing - spec = importlib.util.spec_from_file_location(class_name, absolute_module_path) + spec = importlib.util.spec_from_file_location( + class_name, absolute_module_path + ) if spec and spec.loader: mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) @@ -96,7 +100,9 @@ def _load_component(self, name: str, path: str): sys.path.insert(0, self._base_path) try: - logger.debug(f"Importing custom module {module_path}, class {class_name}") + logger.debug( + f"Importing custom module {module_path}, class {class_name}" + ) mod = importlib.import_module(module_path) self._components[name] = getattr(mod, class_name) return @@ -189,7 +195,7 @@ def from_yaml(cls, path: str, yml: dict): ins._path = path ins._base_path = os.path.dirname(path) ins._node = Graph.from_yaml(ins.factory, yml["node"]) - ins._node.stopped = Event() + ins._node._stopped = Event() return ins @classmethod diff --git a/flyde/nodes.py b/flyde/nodes.py index bfb4b6c..94cc21e 100644 --- a/flyde/nodes.py +++ b/flyde/nodes.py @@ -12,6 +12,7 @@ class InlineValue(Component): """InlineValue sends a constant value to output.""" + icon = "pencil" outputs = {"value": Output(description="The constant value")} def __init__(self, **kwargs): @@ -68,6 +69,7 @@ def __init__(self, config: dict[str, Union[InputConfig, _ConditionConfig]]): class Conditional(Component): """Conditional component evaluates a condition against the input and sends the result to output.""" + icon = "circle-question" inputs = { "leftOperand": Input(description="Left operand of the condition"), "rightOperand": Input(description="Right operand of the condition"), @@ -82,7 +84,11 @@ def parse_config(self, config: dict[str, Any]) -> dict[str, Any]: result = super().parse_config(config) # type: ignore # Handle the condition special case - if "condition" in result and isinstance(result["condition"], dict) and "type" in result["condition"]: + if ( + "condition" in result + and isinstance(result["condition"], dict) + and "type" in result["condition"] + ): result["condition"] = _ConditionConfig(**result["condition"]) return result @@ -90,10 +96,16 @@ def parse_config(self, config: dict[str, Any]) -> dict[str, Any]: def __init__(self, **kwargs): super().__init__(**kwargs) self._config = _ConditionalConfig(self._config) - if hasattr(self._config, "left_operand") and self._config.left_operand.type != InputType.DYNAMIC: + if ( + hasattr(self._config, "left_operand") + and self._config.left_operand.type != InputType.DYNAMIC + ): self.inputs["leftOperand"]._input_mode = InputMode.STATIC self.inputs["leftOperand"].value = self._config.left_operand.value - if hasattr(self._config, "right_operand") and self._config.right_operand.type != InputType.DYNAMIC: + if ( + hasattr(self._config, "right_operand") + and self._config.right_operand.type != InputType.DYNAMIC + ): self.inputs["rightOperand"]._input_mode = InputMode.STATIC self.inputs["rightOperand"].value = self._config.right_operand.value @@ -111,7 +123,9 @@ def _evaluate(self, left_operand: Any, right_operand: Any) -> bool: m = re.match(right_operand, left_operand) return m is not None elif condition_type == _ConditionType.Exists: - return left_operand is not None and left_operand != "" and left_operand != [] + return ( + left_operand is not None and left_operand != "" and left_operand != [] + ) elif condition_type == _ConditionType.NotExists: return left_operand is None or left_operand == "" or left_operand == [] else: @@ -129,6 +143,7 @@ def process(self, leftOperand: Any, rightOperand: Any): class GetAttribute(Component): """Get an attribute from an object or dictionary.""" + icon = "fa-magnifying-glass" inputs = { "object": Input(description="The object or dictionary"), "key": Input(description="The attribute name", type=str), @@ -167,12 +182,21 @@ def process(self, object: Any, key: str): class Http(Component): """Http component makes HTTP requests with urllib.""" + icon = "globe" inputs = { "url": Input(description="URL to request", required=Requiredness.REQUIRED), - "method": Input(description="HTTP method", type=str, required=Requiredness.REQUIRED), - "headers": Input(description="HTTP headers", type=dict, required=Requiredness.OPTIONAL), - "params": Input(description="URL parameters", type=dict, required=Requiredness.OPTIONAL), - "data": Input(description="Request body", type=dict, required=Requiredness.OPTIONAL), + "method": Input( + description="HTTP method", type=str, required=Requiredness.REQUIRED + ), + "headers": Input( + description="HTTP headers", type=dict, required=Requiredness.OPTIONAL + ), + "params": Input( + description="URL parameters", type=dict, required=Requiredness.OPTIONAL + ), + "data": Input( + description="Request body", type=dict, required=Requiredness.OPTIONAL + ), } outputs = { "data": Output(description="Response data"), @@ -198,7 +222,9 @@ def __init__(self, **kwargs): self.inputs["url"]._input_mode = InputMode.STATIC self.inputs["url"].value = self._config["url"].value - if "headers" in self._config and isinstance(self._config["headers"], InputConfig): + if "headers" in self._config and isinstance( + self._config["headers"], InputConfig + ): if self._config["headers"].type == InputType.DYNAMIC: self.inputs["headers"]._input_mode = InputMode.STICKY else: @@ -258,34 +284,39 @@ def process( req.add_header("Content-Length", str(len(data_bytes))) with request.urlopen(req, data=data_bytes) as response: - content_type = response.headers.get('Content-Type', '') + content_type = response.headers.get("Content-Type", "") response_data = response.read() - + # Handle text-based responses - if 'text/' in content_type or 'json' in content_type or 'xml' in content_type or 'application/javascript' in content_type: + if ( + "text/" in content_type + or "json" in content_type + or "xml" in content_type + or "application/javascript" in content_type + ): # Extract charset from content-type header if present - charset = 'utf-8' # Default charset - if 'charset=' in content_type: - charset_part = content_type.split('charset=')[1] - if ';' in charset_part: - charset = charset_part.split(';')[0].strip() + charset = "utf-8" # Default charset + if "charset=" in content_type: + charset_part = content_type.split("charset=")[1] + if ";" in charset_part: + charset = charset_part.split(";")[0].strip() else: charset = charset_part.strip() - + try: response_data = response_data.decode(charset) except (UnicodeDecodeError, LookupError): # Fallback to utf-8 if specified charset fails - response_data = response_data.decode('utf-8', errors='replace') - + response_data = response_data.decode("utf-8", errors="replace") + # Try to parse JSON if the content type indicates JSON - if 'json' in content_type: + if "json" in content_type: try: response_data = json.loads(response_data) except json.JSONDecodeError: pass # Binary data remains as bytes - + self.send("data", response_data) except error.HTTPError as e: raise e diff --git a/tests/.flyde-nodes.json b/tests/.flyde-nodes.json index 15678b0..c4ab2b3 100644 --- a/tests/.flyde-nodes.json +++ b/tests/.flyde-nodes.json @@ -5,6 +5,7 @@ "type": "code", "displayName": "Capitalize", "description": "A component that capitalizes the input.", + "icon": "fa-solid fa-user", "source": { "type": "custom", "data": "custom://components.py/Capitalize" @@ -25,17 +26,16 @@ }, "editorConfig": { "type": "structured" - }, - "icon": "fa-solid fa-user" + } }, - "config": {}, - "icon": "fa-solid fa-user" + "config": {} }, "Echo": { "id": "Echo", "type": "code", "displayName": "Echo", "description": "A simple component that echoes the input.", + "icon": "fa-solid fa-user", "source": { "type": "custom", "data": "custom://components.py/Echo" @@ -56,17 +56,16 @@ }, "editorConfig": { "type": "structured" - }, - "icon": "fa-solid fa-user" + } }, - "config": {}, - "icon": "fa-solid fa-user" + "config": {} }, "Format": { "id": "Format", "type": "code", "displayName": "Format", "description": "Formats the input value with a given format string and sends it to out.", + "icon": "fa-solid fa-user", "source": { "type": "custom", "data": "custom://components.py/Format" @@ -90,17 +89,16 @@ }, "editorConfig": { "type": "structured" - }, - "icon": "fa-solid fa-user" + } }, - "config": {}, - "icon": "fa-solid fa-user" + "config": {} }, "RepeatWordNTimes": { "id": "RepeatWordNTimes", "type": "code", "displayName": "Repeat Word NTimes", "description": "A component that has both inputs and outputs and a sticky input.", + "icon": "fa-solid fa-user", "source": { "type": "custom", "data": "custom://components.py/RepeatWordNTimes" @@ -124,17 +122,16 @@ }, "editorConfig": { "type": "structured" - }, - "icon": "fa-solid fa-user" + } }, - "config": {}, - "icon": "fa-solid fa-user" + "config": {} }, "CustomBob": { "id": "CustomBob", "type": "code", "displayName": "Custom Bob", "description": "A custom external node named Bob", + "icon": "fa-solid fa-user", "source": { "type": "custom", "data": "custom://test_cli.py/CustomBob" @@ -155,24 +152,23 @@ }, "editorConfig": { "type": "structured" - }, - "icon": "fa-solid fa-user" + } }, - "config": {}, - "icon": "fa-solid fa-user" + "config": {} }, "InlineValue": { "id": "InlineValue", "type": "code", - "displayName": "Overridden Inline Value", + "displayName": "Inline Value", "description": "This overrides the standard InlineValue node", + "icon": "fa-solid fa-user", "source": { "type": "package", "data": "@flyde/nodes" }, "editorNode": { "id": "InlineValue", - "displayName": "Overridden Inline Value", + "displayName": "Inline Value", "description": "This overrides the standard InlineValue node", "inputs": {}, "outputs": { @@ -191,6 +187,7 @@ "type": "code", "displayName": "Test Custom Component", "description": "A test component for unit testing.", + "icon": "fa-solid fa-user", "source": { "type": "custom", "data": "custom://test_cli.py/TestCustomComponent" @@ -214,17 +211,16 @@ }, "editorConfig": { "type": "structured" - }, - "icon": "fa-solid fa-user" + } }, - "config": {}, - "icon": "fa-solid fa-user" + "config": {} }, "AllStickyInputsComponent": { "id": "AllStickyInputsComponent", "type": "code", "displayName": "All Sticky Inputs Component", "description": "A component with only sticky inputs to test single execution.", + "icon": "fa-solid fa-user", "source": { "type": "custom", "data": "custom://test_node.py/AllStickyInputsComponent" @@ -248,17 +244,16 @@ }, "editorConfig": { "type": "structured" - }, - "icon": "fa-solid fa-user" + } }, - "config": {}, - "icon": "fa-solid fa-user" + "config": {} }, "CustomRunComponent": { "id": "CustomRunComponent", "type": "code", "displayName": "Custom Run Component", "description": "A component that has a custom run and shutdown handlers.", + "icon": "fa-solid fa-user", "source": { "type": "custom", "data": "custom://test_node.py/CustomRunComponent" @@ -279,17 +274,16 @@ }, "editorConfig": { "type": "structured" - }, - "icon": "fa-solid fa-user" + } }, - "config": {}, - "icon": "fa-solid fa-user" + "config": {} }, "InvalidSendProcess": { "id": "InvalidSendProcess", "type": "code", "displayName": "Invalid Send Process", "description": "A component that tries to send a message without a corresponding output.", + "icon": "fa-solid fa-user", "source": { "type": "custom", "data": "custom://test_node.py/InvalidSendProcess" @@ -306,17 +300,16 @@ "outputs": {}, "editorConfig": { "type": "structured" - }, - "icon": "fa-solid fa-user" + } }, - "config": {}, - "icon": "fa-solid fa-user" + "config": {} }, "NoProcessComponent": { "id": "NoProcessComponent", "type": "code", "displayName": "No Process Component", "description": "A component to test no inputs, outputs, and no process method.", + "icon": "fa-solid fa-user", "source": { "type": "custom", "data": "custom://test_node.py/NoProcessComponent" @@ -329,17 +322,16 @@ "outputs": {}, "editorConfig": { "type": "structured" - }, - "icon": "fa-solid fa-user" + } }, - "config": {}, - "icon": "fa-solid fa-user" + "config": {} }, "SinkComponent": { "id": "SinkComponent", "type": "code", "displayName": "Sink Component", "description": "A component that only has inputs.", + "icon": "fa-solid fa-user", "source": { "type": "custom", "data": "custom://test_node.py/SinkComponent" @@ -359,17 +351,16 @@ "outputs": {}, "editorConfig": { "type": "structured" - }, - "icon": "fa-solid fa-user" + } }, - "config": {}, - "icon": "fa-solid fa-user" + "config": {} }, "SourceComponent": { "id": "SourceComponent", "type": "code", "displayName": "Source Component", "description": "A component that only has outputs.", + "icon": "fa-solid fa-user", "source": { "type": "custom", "data": "custom://test_node.py/SourceComponent" @@ -386,11 +377,120 @@ }, "editorConfig": { "type": "structured" + } + }, + "config": {} + }, + "Conditional": { + "id": "Conditional", + "type": "code", + "displayName": "Conditional", + "description": "Conditional component evaluates a condition against the input and sends the result to output.", + "icon": "circle-question", + "source": { + "type": "package", + "data": "@flyde/nodes" + }, + "editorNode": { + "id": "Conditional", + "displayName": "Conditional", + "description": "Conditional component evaluates a condition against the input and sends the result to output.", + "inputs": { + "leftOperand": { + "description": "Left operand of the condition" + }, + "rightOperand": { + "description": "Right operand of the condition" + } + }, + "outputs": { + "true": { + "description": "Output when the condition is true" + }, + "false": { + "description": "Output when the condition is false" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "GetAttribute": { + "id": "GetAttribute", + "type": "code", + "displayName": "Get Attribute", + "description": "Get an attribute from an object or dictionary.", + "icon": "fa-magnifying-glass", + "source": { + "type": "package", + "data": "@flyde/nodes" + }, + "editorNode": { + "id": "GetAttribute", + "displayName": "Get Attribute", + "description": "Get an attribute from an object or dictionary.", + "inputs": { + "object": { + "description": "The object or dictionary" + }, + "key": { + "description": "The attribute name" + } + }, + "outputs": { + "value": { + "description": "The attribute value" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "Http": { + "id": "Http", + "type": "code", + "displayName": "Http", + "description": "Http component makes HTTP requests with urllib.", + "icon": "globe", + "source": { + "type": "package", + "data": "@flyde/nodes" + }, + "editorNode": { + "id": "Http", + "displayName": "Http", + "description": "Http component makes HTTP requests with urllib.", + "inputs": { + "url": { + "description": "URL to request" + }, + "method": { + "description": "HTTP method" + }, + "headers": { + "description": "HTTP headers" + }, + "params": { + "description": "URL parameters" + }, + "data": { + "description": "Request body" + } + }, + "outputs": { + "data": { + "description": "Response data" + } }, - "icon": "fa-solid fa-user" + "editorConfig": { + "type": "structured" + } }, - "config": {}, - "icon": "fa-solid fa-user" + "config": {} } }, "groups": [ @@ -402,6 +502,7 @@ "Format", "RepeatWordNTimes", "CustomBob", + "InlineValue", "TestCustomComponent", "AllStickyInputsComponent", "CustomRunComponent", @@ -414,6 +515,9 @@ { "title": "Overridden Stdlib", "nodeIds": [ + "Conditional", + "GetAttribute", + "Http", "InlineValue" ] } diff --git a/tests/test_cli.py b/tests/test_cli.py index bca9fbe..4b140d7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,8 +1,8 @@ import json import os +import shutil import tempfile import unittest -from unittest.mock import patch from flyde.cli import ( collect_components_from_directory, @@ -12,7 +12,7 @@ is_stdlib_node, ) from flyde.io import Input, Output -from flyde.node import Component +from flyde.node import Component, SUPPORTED_MACROS class TestCLIHelpers(unittest.TestCase): @@ -34,8 +34,6 @@ def test_convert_class_name_to_display_name(self): self.assertEqual(result, expected) def test_is_stdlib_node(self): - from flyde.node import SUPPORTED_MACROS - # Test that all supported macros are detected as stdlib nodes for macro in SUPPORTED_MACROS: with self.subTest(node_name=macro): @@ -100,14 +98,18 @@ def test_generate_custom_node_json(self): "displayName": "Custom Bob", "description": "A custom external node named Bob", "icon": "fa-solid fa-user", - "source": {"type": "custom", "data": "custom://test_components.py/CustomBob"}, + "source": { + "type": "custom", + "data": "custom://test_components.py/CustomBob", + }, "editorNode": { "id": "CustomBob", "displayName": "Custom Bob", "description": "A custom external node named Bob", - "icon": "fa-solid fa-user", "inputs": {"value": {"description": "Input value to process"}}, - "outputs": {"result": {"description": "Processed result from external runtime"}}, + "outputs": { + "result": {"description": "Processed result from external runtime"} + }, "editorConfig": {"type": "structured"}, }, "config": {}, @@ -115,18 +117,18 @@ def test_generate_custom_node_json(self): self.assertEqual(result, expected) - def test_generate_stdlib_override_json(self): + def test_generate_stdlib_node_json(self): result = generate_node_json("InlineValue", InlineValue, "test_components.py") - expected = { "id": "InlineValue", "type": "code", - "displayName": "Overridden Inline Value", + "displayName": "Inline Value", "description": "This overrides the standard InlineValue node", + "icon": "fa-solid fa-user", "source": {"type": "package", "data": "@flyde/nodes"}, "editorNode": { "id": "InlineValue", - "displayName": "Overridden Inline Value", + "displayName": "Inline Value", "description": "This overrides the standard InlineValue node", "inputs": {}, "outputs": {"value": {"description": "The overridden value"}}, @@ -134,7 +136,6 @@ def test_generate_stdlib_override_json(self): }, "config": {}, } - self.assertEqual(result, expected) def test_generate_node_with_no_docstring(self): @@ -162,8 +163,6 @@ def setUp(self): self.temp_dir = tempfile.mkdtemp() def tearDown(self): - import shutil - shutil.rmtree(self.temp_dir) def test_collect_components_from_directory(self): @@ -247,12 +246,10 @@ def setUp(self): self.temp_dir = tempfile.mkdtemp() def tearDown(self): - import shutil - shutil.rmtree(self.temp_dir) def test_gen_json_with_mixed_components(self): - # Create test files with both custom and stdlib override components + # Create test files with both custom and stdlib components, but do not override stdlib nodes components_py = ''' from flyde.io import Input, Output from flyde.node import Component @@ -269,10 +266,6 @@ class CustomAlice(Component): "input2": Input(description="Second input") } outputs = {"output": Output(description="Combined output")} - -class InlineValue(Component): - """This overrides the standard InlineValue node""" - outputs = {"value": Output(description="The overridden value")} ''' with open(os.path.join(self.temp_dir, "components.py"), "w") as f: @@ -295,10 +288,13 @@ class InlineValue(Component): # Check nodes nodes = data["nodes"] - self.assertEqual(len(nodes), 3) + # Should have all custom nodes plus all stdlib nodes + expected_nodes = set(["CustomBob", "CustomAlice"] + list(SUPPORTED_MACROS)) + self.assertEqual(set(nodes.keys()), expected_nodes) self.assertIn("CustomBob", nodes) self.assertIn("CustomAlice", nodes) - self.assertIn("InlineValue", nodes) + for stdlib_node in SUPPORTED_MACROS: + self.assertIn(stdlib_node, nodes) # Check CustomBob bob = nodes["CustomBob"] @@ -308,14 +304,6 @@ class InlineValue(Component): self.assertEqual(bob["source"]["data"], "custom://components.py/CustomBob") self.assertIn("icon", bob) - # Check InlineValue (stdlib override) - inline = nodes["InlineValue"] - self.assertEqual(inline["id"], "InlineValue") - self.assertEqual(inline["displayName"], "Overridden Inline Value") - self.assertEqual(inline["source"]["type"], "package") - self.assertEqual(inline["source"]["data"], "@flyde/nodes") - self.assertNotIn("icon", inline) - # Check groups groups = data["groups"] self.assertEqual(len(groups), 2) @@ -325,12 +313,7 @@ class InlineValue(Component): stdlib_group = next(g for g in groups if g["title"] == "Overridden Stdlib") self.assertCountEqual(custom_group["nodeIds"], ["CustomBob", "CustomAlice"]) - self.assertCountEqual(stdlib_group["nodeIds"], ["InlineValue"]) - - # Check CustomAlice has correct path format - alice = nodes["CustomAlice"] - self.assertEqual(alice["source"]["type"], "custom") - self.assertEqual(alice["source"]["data"], "custom://components.py/CustomAlice") + self.assertCountEqual(stdlib_group["nodeIds"], list(SUPPORTED_MACROS)) def test_gen_json_empty_directory(self): # Test with directory containing no components @@ -346,17 +329,26 @@ class NotAComponent: with open(os.path.join(self.temp_dir, "empty.py"), "w") as f: f.write(empty_py) - # Capture stdout to verify the message - with patch("builtins.print") as mock_print: - gen_json(self.temp_dir) - mock_print.assert_any_call(f"No Component subclasses found in directory {self.temp_dir}") + # Generate JSON + gen_json(self.temp_dir) - # Should not create the JSON file + # Should create the JSON file with only stdlib nodes if any exist output_file = os.path.join(self.temp_dir, ".flyde-nodes.json") - self.assertFalse(os.path.exists(output_file)) + self.assertTrue(os.path.exists(output_file)) + + with open(output_file, "r") as f: + data = json.load(f) + self.assertIn("nodes", data) + self.assertIn("groups", data) + # At least one stdlib node should be present if stdlib is available + stdlib_group = next( + (g for g in data["groups"] if g["title"] == "Overridden Stdlib"), None + ) + self.assertIsNotNone(stdlib_group) + self.assertGreater(len(stdlib_group["nodeIds"]), 0) def test_gen_json_only_custom_nodes(self): - # Test with only custom nodes (no stdlib overrides) + # Test with only custom nodes (no stdlib nodes in this directory) components_py = ''' from flyde.io import Input, Output from flyde.node import Component @@ -380,24 +372,16 @@ class CustomNode2(Component): with open(output_file, "r") as f: data = json.load(f) - # Should have one group for custom nodes only + # Should have at least the custom nodes group groups = data["groups"] - self.assertEqual(len(groups), 1) - self.assertEqual(groups[0]["title"], "Custom Runtime Nodes") - self.assertCountEqual(groups[0]["nodeIds"], ["CustomNode1", "CustomNode2"]) - - # Check that custom nodes have correct path format - for node_name in ["CustomNode1", "CustomNode2"]: - node = data["nodes"][node_name] - self.assertEqual(node["source"]["type"], "custom") - self.assertEqual(node["source"]["data"], f"custom://components.py/{node_name}") - - # Ensure no stdlib overrides group exists - for group in groups: - self.assertNotEqual(group["title"], "Overridden Stdlib") - - def test_gen_json_only_stdlib_overrides(self): - # Test with only stdlib override nodes + custom_group = next( + (g for g in groups if g["title"] == "Custom Runtime Nodes"), None + ) + self.assertIsNotNone(custom_group) + self.assertCountEqual(custom_group["nodeIds"], ["CustomNode1", "CustomNode2"]) + + def test_gen_json_only_stdlib_nodes(self): + # Test with only stdlib nodes components_py = ''' from flyde.io import Input, Output from flyde.node import Component @@ -421,8 +405,10 @@ class Conditional(Component): with open(output_file, "r") as f: data = json.load(f) - # Should have one group for stdlib overrides only + # Should have stdlib group groups = data["groups"] - self.assertEqual(len(groups), 1) - self.assertEqual(groups[0]["title"], "Overridden Stdlib") - self.assertCountEqual(groups[0]["nodeIds"], ["InlineValue", "Conditional"]) + stdlib_group = next( + (g for g in groups if g["title"] == "Overridden Stdlib"), None + ) + self.assertIsNotNone(stdlib_group) + self.assertCountEqual(stdlib_group["nodeIds"], list(SUPPORTED_MACROS)) diff --git a/tests/test_flow.py b/tests/test_flow.py index 086994b..550cde4 100644 --- a/tests/test_flow.py +++ b/tests/test_flow.py @@ -145,7 +145,9 @@ def test_custom_component_loading_edge_cases(self): display_name="Test", stopped=None, config={}, - source=InstanceSource(type=InstanceSourceType.CUSTOM, data="custom://tests/components.py/"), + source=InstanceSource( + type=InstanceSourceType.CUSTOM, data="custom://tests/components.py/" + ), ) with self.assertRaises(Exception): flow.create_component("Echo", args) @@ -156,7 +158,10 @@ def test_custom_component_loading_edge_cases(self): display_name="Test", stopped=None, config={}, - source=InstanceSource(type=InstanceSourceType.CUSTOM, data="custom://tests/nonexistent.py/Echo"), + source=InstanceSource( + type=InstanceSourceType.CUSTOM, + data="custom://tests/nonexistent.py/Echo", + ), ) with self.assertRaises(Exception): flow.create_component("Echo", args) @@ -167,7 +172,9 @@ def test_custom_component_loading_edge_cases(self): display_name="Test", stopped=None, config={}, - source=InstanceSource(type=InstanceSourceType.CUSTOM, data="custom://tests/components.py/Echo"), + source=InstanceSource( + type=InstanceSourceType.CUSTOM, data="custom://tests/components.py/Echo" + ), ) component = flow.create_component("Echo", args) self.assertIsNotNone(component) From 8d5cb8c64369b2b6413c883a159df1ea77299279 Mon Sep 17 00:00:00 2001 From: Vladimir Sibirov Date: Sat, 26 Jul 2025 17:32:09 +0200 Subject: [PATCH 13/18] Update `flyde-nodes.json` support for Flyde v1.0.45 --- README.md | 4 +- docs/quickstart.md | 4 +- docs/usage.md | 7 +- .../{.flyde-nodes.json => flyde-nodes.json} | 161 ++--------------- flyde/cli.py | 28 ++- tests/{.flyde-nodes.json => flyde-nodes.json} | 169 ++---------------- tests/test_cli.py | 60 +++---- 7 files changed, 73 insertions(+), 360 deletions(-) rename examples/{.flyde-nodes.json => flyde-nodes.json} (63%) rename tests/{.flyde-nodes.json => flyde-nodes.json} (70%) diff --git a/README.md b/README.md index d5f860a..edcfe00 100644 --- a/README.md +++ b/README.md @@ -61,13 +61,13 @@ Install Flyde VSCode extension from the [marketplace](https://marketplace.visual You can browse the component library in the panel on the right. To see your local components click the "View all" button. They will appear under the "Current project". Note that PyFlyde doesn't implement all of the Flyde's stdlib components, only a few essential ones. -Whenever you change your component library classes or their interfaces, use `pyflyde gen` command to generate `.flyde-nodes.json` definitions, e.g.: +Whenever you change your component library classes or their interfaces, use `pyflyde gen` command to generate `flyde-nodes.json` definitions, e.g.: ```bash pyflyde gen examples/ ``` -This will recursively scan all Python files in the directory and its subdirectories to find PyFlyde components and generate a `.flyde-nodes.json` file with relative paths. Flyde editor needs `.flyde-nodes.json` files in order to "see" your components. +This will recursively scan all Python files in the directory and its subdirectories to find PyFlyde components and generate a `flyde-nodes.json` file with relative paths. Flyde editor needs `flyde-nodes.json` files in order to "see" your components. ### Running a Machine Learning example and creating your first project diff --git a/docs/quickstart.md b/docs/quickstart.md index 67640af..a5cd726 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -37,7 +37,7 @@ where = ["mylib"] # With this line Install the dependencies: ```bash -pip install examples/ +pip install examples/ ``` ## Running the Hello World example @@ -48,7 +48,7 @@ First, generate the component metadata for the examples: pyflyde gen examples/ ``` -This will recursively scan all Python files in the `examples/` directory and generate a `.flyde-nodes.json` file with metadata for all PyFlyde components found. +This will recursively scan all Python files in the `examples/` directory and generate a `flyde-nodes.json` file with metadata for all PyFlyde components found. Then run the example flow: diff --git a/docs/usage.md b/docs/usage.md index e507010..475a8ac 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -16,7 +16,7 @@ pyflyde examples/HelloWorld.flyde ## Generating component definitions for Flyde visual editor -To make your Python nodes appear in the Flyde visual editor, you need to generate `.flyde-nodes.json` metadata files for them. +To make your Python nodes appear in the Flyde visual editor, you need to generate `flyde-nodes.json` metadata files for them. Generate JSON definitions for a directory: @@ -24,15 +24,14 @@ Generate JSON definitions for a directory: pyflyde gen mypackage/ ``` -This will recursively scan all `.py` files in the directory and its subdirectories, then generate a `.flyde-nodes.json` file in the specified directory containing metadata for all PyFlyde components found. The paths in the generated JSON file are relative to the directory containing the `.flyde-nodes.json` file, making the component library portable. +This will recursively scan all `.py` files in the directory and its subdirectories, then generate a `flyde-nodes.json` file in the specified directory containing metadata for all PyFlyde components found. The paths in the generated JSON file are relative to the directory containing the `flyde-nodes.json` file, making the component library portable. For example, if you have components in: - `mypackage/components.py` - `mypackage/utils/helpers.py` -The generated `.flyde-nodes.json` will reference them as: +The generated `flyde-nodes.json` will reference them as: - `custom://components.py/ComponentName` - `custom://utils/helpers.py/HelperComponentName` You should run `pyflyde gen` every time you create new modules containing PyFlyde nodes or whenever you update node signatures (name, description, inputs, outputs, etc.). - diff --git a/examples/.flyde-nodes.json b/examples/flyde-nodes.json similarity index 63% rename from examples/.flyde-nodes.json rename to examples/flyde-nodes.json index bdf83ce..2f90e83 100644 --- a/examples/.flyde-nodes.json +++ b/examples/flyde-nodes.json @@ -5,7 +5,7 @@ "type": "code", "displayName": "Load Dataset", "description": "Loads a dataset from a file into a DataFrame.", - "icon": "fa-solid fa-user", + "icon": "fa-brands fa-python", "source": { "type": "custom", "data": "custom://mylib/dataframe.py/LoadDataset" @@ -35,7 +35,7 @@ "type": "code", "displayName": "Scale", "description": "Scales the features of a dataframe with a scikit-learn StandardScaler.", - "icon": "fa-solid fa-user", + "icon": "fa-brands fa-python", "source": { "type": "custom", "data": "custom://mylib/dataframe.py/Scale" @@ -65,7 +65,7 @@ "type": "code", "displayName": "KMeans Cluster", "description": "Clusters the dataframe using K-means clustering.", - "icon": "fa-solid fa-user", + "icon": "fa-brands fa-python", "source": { "type": "custom", "data": "custom://mylib/kmeans.py/KMeansCluster" @@ -98,7 +98,7 @@ "type": "code", "displayName": "KMeans NClusters", "description": "Finds the optimal number of clusters for K-means clustering using silhouette method.", - "icon": "fa-solid fa-user", + "icon": "fa-brands fa-python", "source": { "type": "custom", "data": "custom://mylib/kmeans.py/KMeansNClusters" @@ -131,7 +131,7 @@ "type": "code", "displayName": "PCA2", "description": "Performs PCA on a dataframe and returns the first two principal components.", - "icon": "fa-solid fa-user", + "icon": "fa-brands fa-python", "source": { "type": "custom", "data": "custom://mylib/kmeans.py/PCA2" @@ -161,7 +161,7 @@ "type": "code", "displayName": "Visualize", "description": "Visualizes the clustered dataframe.", - "icon": "fa-solid fa-user", + "icon": "fa-brands fa-python", "source": { "type": "custom", "data": "custom://mylib/kmeans.py/Visualize" @@ -193,7 +193,7 @@ "type": "code", "displayName": "Concat", "description": "Concatenates two strings.", - "icon": "fa-solid fa-user", + "icon": "fa-brands fa-python", "source": { "type": "custom", "data": "custom://mylib/components.py/Concat" @@ -226,7 +226,7 @@ "type": "code", "displayName": "Print", "description": "Prints the input message to the console.", - "icon": "fa-solid fa-user", + "icon": "fa-brands fa-python", "source": { "type": "custom", "data": "custom://mylib/components.py/Print" @@ -247,147 +247,14 @@ }, "config": {} }, - "Conditional": { - "id": "Conditional", - "type": "code", - "displayName": "Conditional", - "description": "Conditional component evaluates a condition against the input and sends the result to output.", - "icon": "circle-question", - "source": { - "type": "package", - "data": "@flyde/nodes" - }, - "editorNode": { - "id": "Conditional", - "displayName": "Conditional", - "description": "Conditional component evaluates a condition against the input and sends the result to output.", - "inputs": { - "leftOperand": { - "description": "Left operand of the condition" - }, - "rightOperand": { - "description": "Right operand of the condition" - } - }, - "outputs": { - "true": { - "description": "Output when the condition is true" - }, - "false": { - "description": "Output when the condition is false" - } - }, - "editorConfig": { - "type": "structured" - } - }, - "config": {} - }, - "GetAttribute": { - "id": "GetAttribute", - "type": "code", - "displayName": "Get Attribute", - "description": "Get an attribute from an object or dictionary.", - "icon": "fa-magnifying-glass", - "source": { - "type": "package", - "data": "@flyde/nodes" - }, - "editorNode": { - "id": "GetAttribute", - "displayName": "Get Attribute", - "description": "Get an attribute from an object or dictionary.", - "inputs": { - "object": { - "description": "The object or dictionary" - }, - "key": { - "description": "The attribute name" - } - }, - "outputs": { - "value": { - "description": "The attribute value" - } - }, - "editorConfig": { - "type": "structured" - } - }, - "config": {} - }, - "Http": { - "id": "Http", - "type": "code", - "displayName": "Http", - "description": "Http component makes HTTP requests with urllib.", - "icon": "globe", - "source": { - "type": "package", - "data": "@flyde/nodes" - }, - "editorNode": { - "id": "Http", - "displayName": "Http", - "description": "Http component makes HTTP requests with urllib.", - "inputs": { - "url": { - "description": "URL to request" - }, - "method": { - "description": "HTTP method" - }, - "headers": { - "description": "HTTP headers" - }, - "params": { - "description": "URL parameters" - }, - "data": { - "description": "Request body" - } - }, - "outputs": { - "data": { - "description": "Response data" - } - }, - "editorConfig": { - "type": "structured" - } - }, - "config": {} - }, - "InlineValue": { - "id": "InlineValue", - "type": "code", - "displayName": "Inline Value", - "description": "InlineValue sends a constant value to output.", - "icon": "pencil", - "source": { - "type": "package", - "data": "@flyde/nodes" - }, - "editorNode": { - "id": "InlineValue", - "displayName": "Inline Value", - "description": "InlineValue sends a constant value to output.", - "inputs": {}, - "outputs": { - "value": { - "description": "The constant value" - } - }, - "editorConfig": { - "type": "structured" - } - }, - "config": {} - } + "Conditional": "@flyde/nodes", + "GetAttribute": "@flyde/nodes", + "Http": "@flyde/nodes", + "InlineValue": "@flyde/nodes" }, "groups": [ { - "title": "Custom Runtime Nodes", + "title": "Your PyFlyde Nodes", "nodeIds": [ "LoadDataset", "Scale", @@ -400,7 +267,7 @@ ] }, { - "title": "Overridden Stdlib", + "title": "PyFlyde Standard Nodes", "nodeIds": [ "Conditional", "GetAttribute", diff --git a/flyde/cli.py b/flyde/cli.py index 133c8f3..cfbfd4d 100644 --- a/flyde/cli.py +++ b/flyde/cli.py @@ -82,33 +82,29 @@ def collect_components_from_directory(directory_path: str) -> dict: return components -def generate_node_json(node_name: str, component_class, file_path: str = "") -> dict: +def generate_node_json(node_name: str, component_class, file_path: str = "") -> dict[str, object] | str: """Generate JSON structure for a single component.""" # Get node metadata description = (component_class.__doc__ or "").strip() display_name = convert_class_name_to_display_name(node_name) # Use package source for stdlib nodes, custom source for others if is_stdlib_node(node_name): - source = {"type": "package", "data": "@flyde/nodes"} + return "@flyde/nodes" else: source = {"type": "custom", "data": f"custom://{file_path}/{node_name}"} - icon = getattr(component_class, "icon", "fa-solid fa-user") + icon = getattr(component_class, "icon", "fa-brands fa-python") # Build inputs structure inputs = {} if hasattr(component_class, "inputs") and component_class.inputs: for input_name, input_obj in component_class.inputs.items(): - inputs[input_name] = { - "description": input_obj.description or f"{input_name} input" - } + inputs[input_name] = {"description": input_obj.description or f"{input_name} input"} # Build outputs structure outputs = {} if hasattr(component_class, "outputs") and component_class.outputs: for output_name, output_obj in component_class.outputs.items(): - outputs[output_name] = { - "description": output_obj.description or f"{output_name} output" - } + outputs[output_name] = {"description": output_obj.description or f"{output_name} output"} # Build the node structure node_data = { @@ -171,15 +167,15 @@ def gen_json(directory_path: str): # Build groups groups = [] if custom_nodes: - groups.append({"title": "Custom Runtime Nodes", "nodeIds": custom_nodes}) + groups.append({"title": "Your PyFlyde Nodes", "nodeIds": custom_nodes}) if stdlib_nodes: - groups.append({"title": "Overridden Stdlib", "nodeIds": stdlib_nodes}) + groups.append({"title": "PyFlyde Standard Nodes", "nodeIds": stdlib_nodes}) # Build final JSON structure json_data = {"nodes": nodes, "groups": groups} # Write to file - output_file = os.path.join(directory_path, ".flyde-nodes.json") + output_file = os.path.join(directory_path, "flyde-nodes.json") with open(output_file, "w") as f: json.dump(json_data, f, indent=2) @@ -194,7 +190,7 @@ def main(): Examples: flyde.py path/to/MyFlow.flyde # Runs a flow - flyde.py gen path/to/directory/ # Generates .flyde-nodes.json for directory + flyde.py gen path/to/directory/ # Generates flyde-nodes.json for directory """, formatter_class=argparse.RawDescriptionHelpFormatter, ) @@ -209,7 +205,7 @@ def main(): parser.add_argument( "path", type=str, - help='Path to a ".flyde" flow file to run, or a directory to generate .flyde-nodes.json for', + help='Path to a ".flyde" flow file to run, or a directory to generate flyde-nodes.json for', ) args = parser.parse_args() @@ -234,8 +230,6 @@ def main(): if os.path.isdir(args.path): gen_json(args.path) else: - raise ValueError( - f"Path {args.path} is not a directory. Only directory generation is supported." - ) + raise ValueError(f"Path {args.path} is not a directory. Only directory generation is supported.") else: raise ValueError(f"Unknown command: {args.command}") diff --git a/tests/.flyde-nodes.json b/tests/flyde-nodes.json similarity index 70% rename from tests/.flyde-nodes.json rename to tests/flyde-nodes.json index c4ab2b3..e404cfd 100644 --- a/tests/.flyde-nodes.json +++ b/tests/flyde-nodes.json @@ -5,7 +5,7 @@ "type": "code", "displayName": "Capitalize", "description": "A component that capitalizes the input.", - "icon": "fa-solid fa-user", + "icon": "fa-brands fa-python", "source": { "type": "custom", "data": "custom://components.py/Capitalize" @@ -35,7 +35,7 @@ "type": "code", "displayName": "Echo", "description": "A simple component that echoes the input.", - "icon": "fa-solid fa-user", + "icon": "fa-brands fa-python", "source": { "type": "custom", "data": "custom://components.py/Echo" @@ -65,7 +65,7 @@ "type": "code", "displayName": "Format", "description": "Formats the input value with a given format string and sends it to out.", - "icon": "fa-solid fa-user", + "icon": "fa-brands fa-python", "source": { "type": "custom", "data": "custom://components.py/Format" @@ -98,7 +98,7 @@ "type": "code", "displayName": "Repeat Word NTimes", "description": "A component that has both inputs and outputs and a sticky input.", - "icon": "fa-solid fa-user", + "icon": "fa-brands fa-python", "source": { "type": "custom", "data": "custom://components.py/RepeatWordNTimes" @@ -131,7 +131,7 @@ "type": "code", "displayName": "Custom Bob", "description": "A custom external node named Bob", - "icon": "fa-solid fa-user", + "icon": "fa-brands fa-python", "source": { "type": "custom", "data": "custom://test_cli.py/CustomBob" @@ -156,38 +156,13 @@ }, "config": {} }, - "InlineValue": { - "id": "InlineValue", - "type": "code", - "displayName": "Inline Value", - "description": "This overrides the standard InlineValue node", - "icon": "fa-solid fa-user", - "source": { - "type": "package", - "data": "@flyde/nodes" - }, - "editorNode": { - "id": "InlineValue", - "displayName": "Inline Value", - "description": "This overrides the standard InlineValue node", - "inputs": {}, - "outputs": { - "value": { - "description": "The overridden value" - } - }, - "editorConfig": { - "type": "structured" - } - }, - "config": {} - }, + "InlineValue": "@flyde/nodes", "TestCustomComponent": { "id": "TestCustomComponent", "type": "code", "displayName": "Test Custom Component", "description": "A test component for unit testing.", - "icon": "fa-solid fa-user", + "icon": "fa-brands fa-python", "source": { "type": "custom", "data": "custom://test_cli.py/TestCustomComponent" @@ -220,7 +195,7 @@ "type": "code", "displayName": "All Sticky Inputs Component", "description": "A component with only sticky inputs to test single execution.", - "icon": "fa-solid fa-user", + "icon": "fa-brands fa-python", "source": { "type": "custom", "data": "custom://test_node.py/AllStickyInputsComponent" @@ -253,7 +228,7 @@ "type": "code", "displayName": "Custom Run Component", "description": "A component that has a custom run and shutdown handlers.", - "icon": "fa-solid fa-user", + "icon": "fa-brands fa-python", "source": { "type": "custom", "data": "custom://test_node.py/CustomRunComponent" @@ -283,7 +258,7 @@ "type": "code", "displayName": "Invalid Send Process", "description": "A component that tries to send a message without a corresponding output.", - "icon": "fa-solid fa-user", + "icon": "fa-brands fa-python", "source": { "type": "custom", "data": "custom://test_node.py/InvalidSendProcess" @@ -309,7 +284,7 @@ "type": "code", "displayName": "No Process Component", "description": "A component to test no inputs, outputs, and no process method.", - "icon": "fa-solid fa-user", + "icon": "fa-brands fa-python", "source": { "type": "custom", "data": "custom://test_node.py/NoProcessComponent" @@ -331,7 +306,7 @@ "type": "code", "displayName": "Sink Component", "description": "A component that only has inputs.", - "icon": "fa-solid fa-user", + "icon": "fa-brands fa-python", "source": { "type": "custom", "data": "custom://test_node.py/SinkComponent" @@ -360,7 +335,7 @@ "type": "code", "displayName": "Source Component", "description": "A component that only has outputs.", - "icon": "fa-solid fa-user", + "icon": "fa-brands fa-python", "source": { "type": "custom", "data": "custom://test_node.py/SourceComponent" @@ -381,121 +356,13 @@ }, "config": {} }, - "Conditional": { - "id": "Conditional", - "type": "code", - "displayName": "Conditional", - "description": "Conditional component evaluates a condition against the input and sends the result to output.", - "icon": "circle-question", - "source": { - "type": "package", - "data": "@flyde/nodes" - }, - "editorNode": { - "id": "Conditional", - "displayName": "Conditional", - "description": "Conditional component evaluates a condition against the input and sends the result to output.", - "inputs": { - "leftOperand": { - "description": "Left operand of the condition" - }, - "rightOperand": { - "description": "Right operand of the condition" - } - }, - "outputs": { - "true": { - "description": "Output when the condition is true" - }, - "false": { - "description": "Output when the condition is false" - } - }, - "editorConfig": { - "type": "structured" - } - }, - "config": {} - }, - "GetAttribute": { - "id": "GetAttribute", - "type": "code", - "displayName": "Get Attribute", - "description": "Get an attribute from an object or dictionary.", - "icon": "fa-magnifying-glass", - "source": { - "type": "package", - "data": "@flyde/nodes" - }, - "editorNode": { - "id": "GetAttribute", - "displayName": "Get Attribute", - "description": "Get an attribute from an object or dictionary.", - "inputs": { - "object": { - "description": "The object or dictionary" - }, - "key": { - "description": "The attribute name" - } - }, - "outputs": { - "value": { - "description": "The attribute value" - } - }, - "editorConfig": { - "type": "structured" - } - }, - "config": {} - }, - "Http": { - "id": "Http", - "type": "code", - "displayName": "Http", - "description": "Http component makes HTTP requests with urllib.", - "icon": "globe", - "source": { - "type": "package", - "data": "@flyde/nodes" - }, - "editorNode": { - "id": "Http", - "displayName": "Http", - "description": "Http component makes HTTP requests with urllib.", - "inputs": { - "url": { - "description": "URL to request" - }, - "method": { - "description": "HTTP method" - }, - "headers": { - "description": "HTTP headers" - }, - "params": { - "description": "URL parameters" - }, - "data": { - "description": "Request body" - } - }, - "outputs": { - "data": { - "description": "Response data" - } - }, - "editorConfig": { - "type": "structured" - } - }, - "config": {} - } + "Conditional": "@flyde/nodes", + "GetAttribute": "@flyde/nodes", + "Http": "@flyde/nodes" }, "groups": [ { - "title": "Custom Runtime Nodes", + "title": "Your PyFlyde Nodes", "nodeIds": [ "Capitalize", "Echo", @@ -513,7 +380,7 @@ ] }, { - "title": "Overridden Stdlib", + "title": "PyFlyde Standard Nodes", "nodeIds": [ "Conditional", "GetAttribute", diff --git a/tests/test_cli.py b/tests/test_cli.py index 4b140d7..b3f2e06 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -12,7 +12,7 @@ is_stdlib_node, ) from flyde.io import Input, Output -from flyde.node import Component, SUPPORTED_MACROS +from flyde.node import SUPPORTED_MACROS, Component class TestCLIHelpers(unittest.TestCase): @@ -97,7 +97,7 @@ def test_generate_custom_node_json(self): "type": "code", "displayName": "Custom Bob", "description": "A custom external node named Bob", - "icon": "fa-solid fa-user", + "icon": "fa-brands fa-python", "source": { "type": "custom", "data": "custom://test_components.py/CustomBob", @@ -107,9 +107,7 @@ def test_generate_custom_node_json(self): "displayName": "Custom Bob", "description": "A custom external node named Bob", "inputs": {"value": {"description": "Input value to process"}}, - "outputs": { - "result": {"description": "Processed result from external runtime"} - }, + "outputs": {"result": {"description": "Processed result from external runtime"}}, "editorConfig": {"type": "structured"}, }, "config": {}, @@ -119,23 +117,7 @@ def test_generate_custom_node_json(self): def test_generate_stdlib_node_json(self): result = generate_node_json("InlineValue", InlineValue, "test_components.py") - expected = { - "id": "InlineValue", - "type": "code", - "displayName": "Inline Value", - "description": "This overrides the standard InlineValue node", - "icon": "fa-solid fa-user", - "source": {"type": "package", "data": "@flyde/nodes"}, - "editorNode": { - "id": "InlineValue", - "displayName": "Inline Value", - "description": "This overrides the standard InlineValue node", - "inputs": {}, - "outputs": {"value": {"description": "The overridden value"}}, - "editorConfig": {"type": "structured"}, - }, - "config": {}, - } + expected = "@flyde/nodes" self.assertEqual(result, expected) def test_generate_node_with_no_docstring(self): @@ -275,7 +257,7 @@ class CustomAlice(Component): gen_json(self.temp_dir) # Check that the file was created - output_file = os.path.join(self.temp_dir, ".flyde-nodes.json") + output_file = os.path.join(self.temp_dir, "flyde-nodes.json") self.assertTrue(os.path.exists(output_file)) # Load and verify the JSON content @@ -309,9 +291,13 @@ class CustomAlice(Component): self.assertEqual(len(groups), 2) # Find groups by title - custom_group = next(g for g in groups if g["title"] == "Custom Runtime Nodes") - stdlib_group = next(g for g in groups if g["title"] == "Overridden Stdlib") + custom_group = next((g for g in groups if g["title"] == "Your PyFlyde Nodes"), None) + stdlib_group = next((g for g in groups if g["title"] == "PyFlyde Standard Nodes"), None) + self.assertIsNotNone(custom_group) + self.assertIsNotNone(stdlib_group) + if custom_group is None or stdlib_group is None: + return self.assertCountEqual(custom_group["nodeIds"], ["CustomBob", "CustomAlice"]) self.assertCountEqual(stdlib_group["nodeIds"], list(SUPPORTED_MACROS)) @@ -333,7 +319,7 @@ class NotAComponent: gen_json(self.temp_dir) # Should create the JSON file with only stdlib nodes if any exist - output_file = os.path.join(self.temp_dir, ".flyde-nodes.json") + output_file = os.path.join(self.temp_dir, "flyde-nodes.json") self.assertTrue(os.path.exists(output_file)) with open(output_file, "r") as f: @@ -341,10 +327,10 @@ class NotAComponent: self.assertIn("nodes", data) self.assertIn("groups", data) # At least one stdlib node should be present if stdlib is available - stdlib_group = next( - (g for g in data["groups"] if g["title"] == "Overridden Stdlib"), None - ) + stdlib_group = next((g for g in data["groups"] if g["title"] == "PyFlyde Standard Nodes"), None) self.assertIsNotNone(stdlib_group) + if stdlib_group is None: + return self.assertGreater(len(stdlib_group["nodeIds"]), 0) def test_gen_json_only_custom_nodes(self): @@ -368,16 +354,16 @@ class CustomNode2(Component): gen_json(self.temp_dir) - output_file = os.path.join(self.temp_dir, ".flyde-nodes.json") + output_file = os.path.join(self.temp_dir, "flyde-nodes.json") with open(output_file, "r") as f: data = json.load(f) # Should have at least the custom nodes group groups = data["groups"] - custom_group = next( - (g for g in groups if g["title"] == "Custom Runtime Nodes"), None - ) + custom_group = next((g for g in groups if g["title"] == "Your PyFlyde Nodes"), None) self.assertIsNotNone(custom_group) + if custom_group is None: + return self.assertCountEqual(custom_group["nodeIds"], ["CustomNode1", "CustomNode2"]) def test_gen_json_only_stdlib_nodes(self): @@ -401,14 +387,14 @@ class Conditional(Component): gen_json(self.temp_dir) - output_file = os.path.join(self.temp_dir, ".flyde-nodes.json") + output_file = os.path.join(self.temp_dir, "flyde-nodes.json") with open(output_file, "r") as f: data = json.load(f) # Should have stdlib group groups = data["groups"] - stdlib_group = next( - (g for g in groups if g["title"] == "Overridden Stdlib"), None - ) + stdlib_group = next((g for g in groups if g["title"] == "PyFlyde Standard Nodes"), None) self.assertIsNotNone(stdlib_group) + if stdlib_group is None: + return self.assertCountEqual(stdlib_group["nodeIds"], list(SUPPORTED_MACROS)) From 8cbce9b7057a31dcfb73eac5918b77af855d4f84 Mon Sep 17 00:00:00 2001 From: Vladimir Sibirov Date: Sat, 26 Jul 2025 18:19:43 +0200 Subject: [PATCH 14/18] Add `*.flyde` files to flyde-nodes.json --- examples/flyde-nodes.json | 48 ++++++- flyde/cli.py | 101 +++++++++++++- tests/TestFanIn.flyde | 2 +- tests/TestFanInGraph.flyde | 2 +- tests/TestInOutFlow.flyde | 2 +- tests/TestIsolatedFlow.flyde | 2 +- tests/TestStdlib.flyde | 2 +- tests/flyde-nodes.json | 241 ++++++++++++++++++++++++++++++++- tests/test_cli.py | 255 +++++++++++++++++++++++++++++++++++ 9 files changed, 645 insertions(+), 10 deletions(-) diff --git a/examples/flyde-nodes.json b/examples/flyde-nodes.json index 2f90e83..926943d 100644 --- a/examples/flyde-nodes.json +++ b/examples/flyde-nodes.json @@ -247,6 +247,50 @@ }, "config": {} }, + "Clustering": { + "id": "Clustering", + "type": "visual", + "displayName": "Clustering", + "description": "", + "icon": "fa-diagram-project", + "source": { + "type": "file", + "data": "Clustering.flyde" + }, + "editorNode": { + "id": "Clustering", + "displayName": "Clustering", + "description": "", + "inputs": {}, + "outputs": {}, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "HelloPy": { + "id": "HelloPy", + "type": "visual", + "displayName": "Hello Py", + "description": "", + "icon": "fa-diagram-project", + "source": { + "type": "file", + "data": "HelloPy.flyde" + }, + "editorNode": { + "id": "HelloPy", + "displayName": "Hello Py", + "description": "", + "inputs": {}, + "outputs": {}, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, "Conditional": "@flyde/nodes", "GetAttribute": "@flyde/nodes", "Http": "@flyde/nodes", @@ -263,7 +307,9 @@ "PCA2", "Visualize", "Concat", - "Print" + "Print", + "Clustering", + "HelloPy" ] }, { diff --git a/flyde/cli.py b/flyde/cli.py index cfbfd4d..f6261ef 100644 --- a/flyde/cli.py +++ b/flyde/cli.py @@ -10,6 +10,8 @@ import re import sys +import yaml + from flyde.flow import Flow, add_folder_to_path from flyde.node import SUPPORTED_MACROS, Component @@ -74,7 +76,7 @@ def collect_components_from_directory(directory_path: str) -> dict: and issubclass(obj, Component) and obj.__module__ == module_path ): - components[name] = {"class": obj, "file_path": relative_path} + components[name] = {"class": obj, "file_path": relative_path, "type": "python"} except Exception as e: logger.warning(f"Failed to import module from {py_file}: {e}") @@ -82,6 +84,91 @@ def collect_components_from_directory(directory_path: str) -> dict: return components +def collect_flyde_nodes_from_directory(directory_path: str) -> dict: + """Collect all .flyde files from a directory and its subdirectories.""" + flyde_nodes = {} + + # Find all .flyde files in the directory and subdirectories recursively + flyde_files = glob.glob(os.path.join(directory_path, "**", "*.flyde"), recursive=True) + + for flyde_file in flyde_files: + try: + # Get relative path from the directory + relative_path = os.path.relpath(flyde_file, directory_path) + + # Load the YAML content + with open(flyde_file, "r") as f: + flyde_data = yaml.safe_load(f) + + # Extract node information + node_data = flyde_data.get("node", {}) + node_id = os.path.splitext(os.path.basename(flyde_file))[0] + description = node_data.get("description", flyde_data.get("description", "")) + + # Extract inputs and outputs + inputs = node_data.get("inputs", {}) + outputs = node_data.get("outputs", {}) + + flyde_nodes[node_id] = { + "file_path": relative_path, + "description": description, + "inputs": inputs, + "outputs": outputs, + "type": "flyde", + } + + except Exception as e: + logger.warning(f"Failed to parse .flyde file {flyde_file}: {e}") + + return flyde_nodes + + +def generate_flyde_node_json(node_name: str, flyde_info: dict) -> dict: + """Generate JSON structure for a .flyde file node.""" + file_path = flyde_info["file_path"] + description = flyde_info["description"] + inputs = flyde_info["inputs"] + outputs = flyde_info["outputs"] + + display_name = convert_class_name_to_display_name(node_name) + + # Build inputs structure + editor_inputs = {} + for input_name, input_data in inputs.items(): + mode = input_data.get("mode", "required") + input_description = f"{input_name} input" + if mode == "required": + input_description += " (required)" + editor_inputs[input_name] = {"description": input_description} + + # Build outputs structure + editor_outputs = {} + for output_name, output_data in outputs.items(): + output_description = f"{output_name} output" + editor_outputs[output_name] = {"description": output_description} + + # Build the node structure + node_data = { + "id": node_name, + "type": "visual", + "displayName": display_name, + "description": description, + "icon": "fa-diagram-project", + "source": {"type": "file", "data": file_path}, + "editorNode": { + "id": node_name, + "displayName": display_name, + "description": description, + "inputs": editor_inputs, + "outputs": editor_outputs, + "editorConfig": {"type": "structured"}, + }, + "config": {}, + } + + return node_data + + def generate_node_json(node_name: str, component_class, file_path: str = "") -> dict[str, object] | str: """Generate JSON structure for a single component.""" # Get node metadata @@ -135,13 +222,16 @@ def gen_json(directory_path: str): # Collect all components components = collect_components_from_directory(directory_path) + # Collect all .flyde nodes + flyde_nodes = collect_flyde_nodes_from_directory(directory_path) + # Always include stdlib nodes from flyde/nodes.py stdlib_dir = os.path.join(os.path.dirname(__file__), "nodes.py") stdlib_components = collect_components_from_directory(os.path.dirname(stdlib_dir)) stdlib_node_names = [name for name in stdlib_components if is_stdlib_node(name)] - if not components and not stdlib_node_names: - print(f"No Component subclasses found in directory {directory_path} or stdlib") + if not components and not flyde_nodes and not stdlib_node_names: + print(f"No Component subclasses or .flyde files found in directory {directory_path} or stdlib") return # Build nodes structure @@ -156,6 +246,11 @@ def gen_json(directory_path: str): nodes[node_name] = generate_node_json(node_name, component_class, file_path) custom_nodes.append(node_name) + # Add .flyde nodes + for node_name, flyde_info in flyde_nodes.items(): + nodes[node_name] = generate_flyde_node_json(node_name, flyde_info) + custom_nodes.append(node_name) + # Add stdlib nodes (from flyde/nodes.py) as custom nodes, but group as stdlib overrides for node_name in stdlib_node_names: if node_name not in nodes: diff --git a/tests/TestFanIn.flyde b/tests/TestFanIn.flyde index ddf9114..25bcbd0 100644 --- a/tests/TestFanIn.flyde +++ b/tests/TestFanIn.flyde @@ -106,7 +106,7 @@ node: to: insId: __this pinId: out - id: Example + id: TestFanIn inputs: str: mode: required diff --git a/tests/TestFanInGraph.flyde b/tests/TestFanInGraph.flyde index 3a0bcad..fe50aad 100644 --- a/tests/TestFanInGraph.flyde +++ b/tests/TestFanInGraph.flyde @@ -102,7 +102,7 @@ node: to: insId: __this pinId: out - id: Example + id: TestFanInGraph inputs: str: mode: required diff --git a/tests/TestInOutFlow.flyde b/tests/TestInOutFlow.flyde index d3abab8..d463964 100644 --- a/tests/TestInOutFlow.flyde +++ b/tests/TestInOutFlow.flyde @@ -108,7 +108,7 @@ node: to: insId: Format-ve0397r pinId: format - id: Example + id: TestInOutFlow inputs: inMsg: mode: required diff --git a/tests/TestIsolatedFlow.flyde b/tests/TestIsolatedFlow.flyde index 9d7fdee..971b44a 100644 --- a/tests/TestIsolatedFlow.flyde +++ b/tests/TestIsolatedFlow.flyde @@ -36,7 +36,7 @@ node: to: insId: Echo-x8049tw pinId: inp - id: Example + id: TestIsolatedFlow inputs: {} outputs: {} inputsPosition: {} diff --git a/tests/TestStdlib.flyde b/tests/TestStdlib.flyde index 1b71f14..9b88c04 100644 --- a/tests/TestStdlib.flyde +++ b/tests/TestStdlib.flyde @@ -36,7 +36,7 @@ node: to: insId: __this pinId: response - id: Example + id: TestStdlib inputs: {} outputs: response: diff --git a/tests/flyde-nodes.json b/tests/flyde-nodes.json index e404cfd..bcb21aa 100644 --- a/tests/flyde-nodes.json +++ b/tests/flyde-nodes.json @@ -356,6 +356,237 @@ }, "config": {} }, + "TestStdlib": { + "id": "TestStdlib", + "type": "visual", + "displayName": "Test Stdlib", + "description": "", + "icon": "fa-diagram-project", + "source": { + "type": "file", + "data": "TestStdlib.flyde" + }, + "editorNode": { + "id": "TestStdlib", + "displayName": "Test Stdlib", + "description": "", + "inputs": {}, + "outputs": { + "response": { + "description": "response output" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "Repeat3Times": { + "id": "Repeat3Times", + "type": "visual", + "displayName": "Repeat3Times", + "description": "For each input string, sends a string with the same content repeated 3 times", + "icon": "fa-diagram-project", + "source": { + "type": "file", + "data": "Repeat3Times.flyde" + }, + "editorNode": { + "id": "Repeat3Times", + "displayName": "Repeat3Times", + "description": "For each input string, sends a string with the same content repeated 3 times", + "inputs": { + "word": { + "description": "word input (required)" + } + }, + "outputs": { + "word3x": { + "description": "word3x output" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "TestNestedFlow": { + "id": "TestNestedFlow", + "type": "visual", + "displayName": "Test Nested Flow", + "description": "Repeats input 3xN times", + "icon": "fa-diagram-project", + "source": { + "type": "file", + "data": "TestNestedFlow.flyde" + }, + "editorNode": { + "id": "TestNestedFlow", + "displayName": "Test Nested Flow", + "description": "Repeats input 3xN times", + "inputs": { + "inp": { + "description": "inp input (required)" + }, + "n": { + "description": "n input (required)" + } + }, + "outputs": { + "out": { + "description": "out output" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "TestFanInGraph": { + "id": "TestFanInGraph", + "type": "visual", + "displayName": "Test Fan In Graph", + "description": "", + "icon": "fa-diagram-project", + "source": { + "type": "file", + "data": "TestFanInGraph.flyde" + }, + "editorNode": { + "id": "TestFanInGraph", + "displayName": "Test Fan In Graph", + "description": "", + "inputs": { + "str": { + "description": "str input (required)" + } + }, + "outputs": { + "out": { + "description": "out output" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "TestFanIn": { + "id": "TestFanIn", + "type": "visual", + "displayName": "Test Fan In", + "description": "", + "icon": "fa-diagram-project", + "source": { + "type": "file", + "data": "TestFanIn.flyde" + }, + "editorNode": { + "id": "TestFanIn", + "displayName": "Test Fan In", + "description": "", + "inputs": { + "str": { + "description": "str input (required)" + } + }, + "outputs": { + "out": { + "description": "out output" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "TestIsolatedFlow": { + "id": "TestIsolatedFlow", + "type": "visual", + "displayName": "Test Isolated Flow", + "description": "", + "icon": "fa-diagram-project", + "source": { + "type": "file", + "data": "TestIsolatedFlow.flyde" + }, + "editorNode": { + "id": "TestIsolatedFlow", + "displayName": "Test Isolated Flow", + "description": "", + "inputs": {}, + "outputs": {}, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "TestCustomLoad": { + "id": "TestCustomLoad", + "type": "visual", + "displayName": "Test Custom Load", + "description": "Test flow that loads custom components using custom:// source format", + "icon": "fa-diagram-project", + "source": { + "type": "file", + "data": "TestCustomLoad.flyde" + }, + "editorNode": { + "id": "TestCustomLoad", + "displayName": "Test Custom Load", + "description": "Test flow that loads custom components using custom:// source format", + "inputs": { + "input": { + "description": "input input (required)" + } + }, + "outputs": { + "output": { + "description": "output output" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "TestInOutFlow": { + "id": "TestInOutFlow", + "type": "visual", + "displayName": "Test In Out Flow", + "description": "", + "icon": "fa-diagram-project", + "source": { + "type": "file", + "data": "TestInOutFlow.flyde" + }, + "editorNode": { + "id": "TestInOutFlow", + "displayName": "Test In Out Flow", + "description": "", + "inputs": { + "inMsg": { + "description": "inMsg input (required)" + } + }, + "outputs": { + "outMsg": { + "description": "outMsg output" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, "Conditional": "@flyde/nodes", "GetAttribute": "@flyde/nodes", "Http": "@flyde/nodes" @@ -376,7 +607,15 @@ "InvalidSendProcess", "NoProcessComponent", "SinkComponent", - "SourceComponent" + "SourceComponent", + "TestStdlib", + "Repeat3Times", + "TestNestedFlow", + "TestFanInGraph", + "TestFanIn", + "TestIsolatedFlow", + "TestCustomLoad", + "TestInOutFlow" ] }, { diff --git a/tests/test_cli.py b/tests/test_cli.py index b3f2e06..7a43f0f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -6,8 +6,10 @@ from flyde.cli import ( collect_components_from_directory, + collect_flyde_nodes_from_directory, convert_class_name_to_display_name, gen_json, + generate_flyde_node_json, generate_node_json, is_stdlib_node, ) @@ -223,6 +225,170 @@ def test_collect_components_empty_directory(self): self.assertEqual(len(components), 0) +class TestCollectFlydeNodes(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.temp_dir) + + def test_collect_flyde_nodes_from_directory(self): + # Use existing test files from the tests directory + test_files_dir = os.path.join(os.path.dirname(__file__), "..") + flyde_nodes = collect_flyde_nodes_from_directory(os.path.join(test_files_dir, "tests")) + + # Should find several .flyde nodes + self.assertGreater(len(flyde_nodes), 0) + self.assertIn("TestNestedFlow", flyde_nodes) + self.assertIn("Repeat3Times", flyde_nodes) + + # Check TestNestedFlow structure (uses filename as ID) + nested_flow = flyde_nodes["TestNestedFlow"] + self.assertEqual(nested_flow["type"], "flyde") + self.assertEqual(nested_flow["description"], "Repeats input 3xN times") + self.assertIn("file_path", nested_flow) + self.assertEqual(len(nested_flow["inputs"]), 2) + self.assertEqual(len(nested_flow["outputs"]), 1) + self.assertEqual(nested_flow["inputs"]["inp"]["mode"], "required") + self.assertEqual(nested_flow["inputs"]["n"]["mode"], "required") + + # Check Repeat3Times structure + repeat_flow = flyde_nodes["Repeat3Times"] + self.assertEqual(repeat_flow["type"], "flyde") + self.assertIn("For each input string", repeat_flow["description"]) + self.assertEqual(len(repeat_flow["inputs"]), 1) + self.assertEqual(len(repeat_flow["outputs"]), 1) + + def test_collect_flyde_nodes_invalid_yaml(self): + # Create a file with invalid YAML syntax + invalid_flyde = """ +This is not valid YAML syntax! +node: + inputs: + - invalid structure +""" + + with open(os.path.join(self.temp_dir, "invalid.flyde"), "w") as f: + f.write(invalid_flyde) + + # Should handle the error gracefully + flyde_nodes = collect_flyde_nodes_from_directory(self.temp_dir) + self.assertEqual(len(flyde_nodes), 0) + + def test_collect_flyde_nodes_empty_directory(self): + # Test with empty directory + flyde_nodes = collect_flyde_nodes_from_directory(self.temp_dir) + self.assertEqual(len(flyde_nodes), 0) + + def test_collect_flyde_nodes_uses_filename_as_id(self): + # Test that node ID always comes from filename, not YAML id field + test_flyde = """imports: {} +node: + instances: [] + connections: [] + id: DummyExample + inputs: + inp: + mode: required + outputs: + out: + delayed: false + inputsPosition: {} + outputsPosition: {} +description: Test that uses filename for ID +""" + + with open(os.path.join(self.temp_dir, "ActualNodeName.flyde"), "w") as f: + f.write(test_flyde) + + flyde_nodes = collect_flyde_nodes_from_directory(self.temp_dir) + self.assertEqual(len(flyde_nodes), 1) + # Should use filename, not the "DummyExample" from YAML + self.assertIn("ActualNodeName", flyde_nodes) + self.assertNotIn("DummyExample", flyde_nodes) + + def test_collect_flyde_nodes_description_priority(self): + # Test that node.description takes priority over root description + test_flyde_node_desc = """imports: {} +node: + instances: [] + connections: [] + inputs: + inp: + mode: required + outputs: + out: + delayed: false + inputsPosition: {} + outputsPosition: {} + description: Node level description +description: Root level description +""" + + with open(os.path.join(self.temp_dir, "TestPriority.flyde"), "w") as f: + f.write(test_flyde_node_desc) + + flyde_nodes = collect_flyde_nodes_from_directory(self.temp_dir) + self.assertEqual(len(flyde_nodes), 1) + self.assertIn("TestPriority", flyde_nodes) + # Should prefer node.description over root description + self.assertEqual(flyde_nodes["TestPriority"]["description"], "Node level description") + + +class TestGenerateFlydeNodeJson(unittest.TestCase): + def test_generate_flyde_node_json(self): + self.maxDiff = None + flyde_info = { + "file_path": "test_flows/TestFlow.flyde", + "description": "A test flow for unit testing", + "inputs": {"input1": {"mode": "required"}, "input2": {"mode": "optional"}}, + "outputs": {"output1": {}, "output2": {}}, + "type": "flyde", + } + + result = generate_flyde_node_json("TestFlow", flyde_info) + + expected = { + "id": "TestFlow", + "type": "visual", + "displayName": "Test Flow", + "description": "A test flow for unit testing", + "icon": "fa-diagram-project", + "source": {"type": "file", "data": "test_flows/TestFlow.flyde"}, + "editorNode": { + "id": "TestFlow", + "displayName": "Test Flow", + "description": "A test flow for unit testing", + "inputs": { + "input1": {"description": "input1 input (required)"}, + "input2": {"description": "input2 input"}, + }, + "outputs": { + "output1": {"description": "output1 output"}, + "output2": {"description": "output2 output"}, + }, + "editorConfig": {"type": "structured"}, + }, + "config": {}, + } + + self.assertEqual(result, expected) + + def test_generate_flyde_node_with_no_inputs_outputs(self): + flyde_info = { + "file_path": "EmptyFlow.flyde", + "description": "An empty flow", + "inputs": {}, + "outputs": {}, + "type": "flyde", + } + + result = generate_flyde_node_json("EmptyFlow", flyde_info) + + self.assertEqual(result["editorNode"]["inputs"], {}) + self.assertEqual(result["editorNode"]["outputs"], {}) + + class TestGenJson(unittest.TestCase): def setUp(self): self.temp_dir = tempfile.mkdtemp() @@ -301,6 +467,95 @@ class CustomAlice(Component): self.assertCountEqual(custom_group["nodeIds"], ["CustomBob", "CustomAlice"]) self.assertCountEqual(stdlib_group["nodeIds"], list(SUPPORTED_MACROS)) + def test_gen_json_with_flyde_files(self): + # Create test files with both .py components and copy an existing .flyde file + components_py = ''' +from flyde.io import Input, Output +from flyde.node import Component + +class CustomNode(Component): + """A custom Python node""" + inputs = {"value": Input(description="Input value")} + outputs = {"result": Output(description="Result value")} +''' + + with open(os.path.join(self.temp_dir, "components.py"), "w") as f: + f.write(components_py) + + # Copy an existing .flyde file + test_flyde_source = os.path.join(os.path.dirname(__file__), "..", "tests", "Repeat3Times.flyde") + test_flyde_dest = os.path.join(self.temp_dir, "TestFlow.flyde") + shutil.copy(test_flyde_source, test_flyde_dest) + + # Generate JSON + gen_json(self.temp_dir) + + # Check that the file was created + output_file = os.path.join(self.temp_dir, "flyde-nodes.json") + self.assertTrue(os.path.exists(output_file)) + + # Load and verify the JSON content + with open(output_file, "r") as f: + data = json.load(f) + + # Check structure + self.assertIn("nodes", data) + self.assertIn("groups", data) + + # Check nodes - should have both Python and .flyde nodes + nodes = data["nodes"] + expected_custom_nodes = set(["CustomNode", "TestFlow"] + list(SUPPORTED_MACROS)) + self.assertEqual(set(nodes.keys()), expected_custom_nodes) + + # Check CustomNode (Python) + custom_node = nodes["CustomNode"] + self.assertEqual(custom_node["type"], "code") + self.assertEqual(custom_node["source"]["type"], "custom") + + # Check TestFlow (.flyde) - uses filename as ID + test_flow = nodes["TestFlow"] + self.assertEqual(test_flow["id"], "TestFlow") + self.assertEqual(test_flow["type"], "visual") + self.assertEqual(test_flow["displayName"], "Test Flow") + self.assertEqual(test_flow["icon"], "fa-diagram-project") + self.assertEqual(test_flow["source"]["type"], "file") + self.assertEqual(test_flow["source"]["data"], "TestFlow.flyde") + + # Check groups + groups = data["groups"] + custom_group = next((g for g in groups if g["title"] == "Your PyFlyde Nodes"), None) + self.assertIsNotNone(custom_group) + if custom_group is not None: + self.assertIn("CustomNode", custom_group["nodeIds"]) + self.assertIn("TestFlow", custom_group["nodeIds"]) + + def test_gen_json_only_flyde_files(self): + # Test with only .flyde files (no Python components) using existing files + test_flyde_source_1 = os.path.join(os.path.dirname(__file__), "..", "tests", "Repeat3Times.flyde") + test_flyde_source_2 = os.path.join(os.path.dirname(__file__), "..", "tests", "TestNestedFlow.flyde") + + shutil.copy(test_flyde_source_1, os.path.join(self.temp_dir, "Flow1.flyde")) + shutil.copy(test_flyde_source_2, os.path.join(self.temp_dir, "Flow2.flyde")) + + gen_json(self.temp_dir) + + output_file = os.path.join(self.temp_dir, "flyde-nodes.json") + with open(output_file, "r") as f: + data = json.load(f) + + # Should have .flyde nodes and stdlib nodes + nodes = data["nodes"] + expected_nodes = set(["Flow1", "Flow2"] + list(SUPPORTED_MACROS)) + self.assertEqual(set(nodes.keys()), expected_nodes) + + # Check that .flyde nodes are in the custom group + groups = data["groups"] + custom_group = next((g for g in groups if g["title"] == "Your PyFlyde Nodes"), None) + self.assertIsNotNone(custom_group) + if custom_group is not None: + self.assertIn("Flow1", custom_group["nodeIds"]) + self.assertIn("Flow2", custom_group["nodeIds"]) + def test_gen_json_empty_directory(self): # Test with directory containing no components empty_py = """ From 64c5cbf750a6a7cfe7a5eac2a551c6e3efef0c13 Mon Sep 17 00:00:00 2001 From: Vladimir Sibirov Date: Sat, 26 Jul 2025 18:22:12 +0200 Subject: [PATCH 15/18] Fix Python 3.9 compat --- flyde/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flyde/cli.py b/flyde/cli.py index f6261ef..54b4586 100644 --- a/flyde/cli.py +++ b/flyde/cli.py @@ -9,6 +9,7 @@ import pprint import re import sys +from typing import Union import yaml @@ -169,7 +170,7 @@ def generate_flyde_node_json(node_name: str, flyde_info: dict) -> dict: return node_data -def generate_node_json(node_name: str, component_class, file_path: str = "") -> dict[str, object] | str: +def generate_node_json(node_name: str, component_class, file_path: str = "") -> Union[dict, str]: """Generate JSON structure for a single component.""" # Get node metadata description = (component_class.__doc__ or "").strip() From 09f1488986441843335777345cb0b8ff1807d412 Mon Sep 17 00:00:00 2001 From: Vladimir Sibirov Date: Sat, 26 Jul 2025 19:26:35 +0200 Subject: [PATCH 16/18] Linter and typing fixes --- Makefile | 7 +- flyde/cli.py | 24 ++++- flyde/cli.pyi | 19 ---- flyde/flow.pyi | 47 -------- flyde/io.pyi | 179 ------------------------------- flyde/node.pyi | 139 ------------------------ flyde/nodes.pyi | 61 ----------- flyde/{__init__.pyi => py.typed} | 0 pyproject.toml | 3 + tests/test_cli.py | 3 + 10 files changed, 29 insertions(+), 453 deletions(-) delete mode 100644 flyde/cli.pyi delete mode 100644 flyde/flow.pyi delete mode 100644 flyde/io.pyi delete mode 100644 flyde/node.pyi delete mode 100644 flyde/nodes.pyi rename flyde/{__init__.pyi => py.typed} (100%) diff --git a/Makefile b/Makefile index 4266a31..1ad9ea9 100644 --- a/Makefile +++ b/Makefile @@ -19,11 +19,6 @@ lint: @black $(LIB_DIR) $(TEST_DIR); @flake8 $(LIB_DIR) $(TEST_DIR); -stubgen: - @echo "Generating type stubs..." - @rm -f $(SRC_DIR)/*.pyi; - @stubgen $(SRC_DIR) --include-docstrings --include-private -o .; - test: @echo "Running tests..." @$(PYTHON) -m unittest discover -s $(TEST_DIR) -p "test_$(if $(mod),$(mod),*).py"; @@ -44,7 +39,7 @@ builddist: @rm -f ./dist/* @$(PYTHON) -m build; -release: lint test stubgen gen builddist +release: lint test gen builddist @echo "Releasing the project..."; upload: diff --git a/flyde/cli.py b/flyde/cli.py index 54b4586..c758a53 100644 --- a/flyde/cli.py +++ b/flyde/cli.py @@ -9,7 +9,7 @@ import pprint import re import sys -from typing import Union +from typing import Any, TypedDict, Union import yaml @@ -21,6 +21,26 @@ logger = logging.getLogger(__name__) +class EditorNodeDict(TypedDict): + id: str + displayName: str + description: str + inputs: dict[str, dict[str, str]] + outputs: dict[str, dict[str, str]] + editorConfig: dict[str, str] + + +class NodeDict(TypedDict): + id: str + type: str + displayName: str + description: str + icon: str + source: dict[str, str] + editorNode: EditorNodeDict + config: dict[str, Any] + + def py_path_to_module(py_path: str) -> str: return py_path.replace("/", ".").replace(".py", "") @@ -170,7 +190,7 @@ def generate_flyde_node_json(node_name: str, flyde_info: dict) -> dict: return node_data -def generate_node_json(node_name: str, component_class, file_path: str = "") -> Union[dict, str]: +def generate_node_json(node_name: str, component_class, file_path: str = "") -> Union[dict[str, Any], str]: """Generate JSON structure for a single component.""" # Get node metadata description = (component_class.__doc__ or "").strip() diff --git a/flyde/cli.pyi b/flyde/cli.pyi deleted file mode 100644 index 925fdb8..0000000 --- a/flyde/cli.pyi +++ /dev/null @@ -1,19 +0,0 @@ -from _typeshed import Incomplete -from flyde.flow import Flow as Flow, add_folder_to_path as add_folder_to_path -from flyde.node import Component as Component, SUPPORTED_MACROS as SUPPORTED_MACROS - -log_level: Incomplete -logger: Incomplete - -def py_path_to_module(py_path: str) -> str: ... -def convert_class_name_to_display_name(class_name: str) -> str: - """Convert a class name like 'MyCustomNode' to 'My Custom Node'.""" -def is_stdlib_node(node_name: str) -> bool: - """Check if a node name matches a stdlib node.""" -def collect_components_from_directory(directory_path: str) -> dict: - """Collect all Component subclasses from .py files in a directory.""" -def generate_node_json(node_name: str, component_class, file_path: str = '') -> dict: - """Generate JSON structure for a single component.""" -def gen_json(directory_path: str): - """Generate JSON file for all components in a directory.""" -def main() -> None: ... diff --git a/flyde/flow.pyi b/flyde/flow.pyi deleted file mode 100644 index 14e2b0b..0000000 --- a/flyde/flow.pyi +++ /dev/null @@ -1,47 +0,0 @@ -from _typeshed import Incomplete -from flyde.node import Graph as Graph, InstanceArgs as InstanceArgs, InstanceType as InstanceType -from threading import Event - -logger: Incomplete - -class Flow: - """Flow is a root-level runnable directed acyclic graph of nodes.""" - _imports: Incomplete - _path: str - _base_path: str - _node: Incomplete - _components: Incomplete - _graphs: Incomplete - def __init__(self, imports: dict[str, list[str]]) -> None: ... - def _preload_imports(self, base_path: str, imports: dict[str, list[str]]): ... - def _load_graph(self, name: str, path: str): - """Loads a graph YAML.""" - def _load_component(self, name: str, path: str): - """Loads a component from a Python module.""" - def create_graph(self, name: str, args: InstanceArgs): ... - def create_component(self, name: str, args: InstanceArgs): ... - def factory(self, class_name: str, args: InstanceArgs): - """Factory method to create a node from a class name and arguments. - - It is used by the runtime to create nodes from the YAML definition or on the fly. - """ - def run(self) -> None: - """Start the flow running. This is a non-blocking call as the flow runs in a separate thread.""" - def run_sync(self) -> None: - """Run the flow synchronously. Shutdown handlers will be executed after the flow has finished.""" - @property - def node(self) -> Graph: - """The root node of the flow.""" - @property - def stopped(self) -> Event: - """Stopped event is set when the flow has finished working.""" - @classmethod - def from_yaml(cls, path: str, yml: dict): - """Load Flyde Flow definition from parsed YAML dict.""" - @classmethod - def from_file(cls, path: str): - """Load Flyde Flow definition from a *.flyde YAML file.""" - def to_dict(self) -> dict: ... - -def add_folder_to_path(path: str): ... -def load_yaml_file(yaml_file: str) -> dict: ... diff --git a/flyde/io.pyi b/flyde/io.pyi deleted file mode 100644 index b841fca..0000000 --- a/flyde/io.pyi +++ /dev/null @@ -1,179 +0,0 @@ -from _typeshed import Incomplete -from dataclasses import dataclass -from enum import Enum -from queue import Queue -from typing import Any - -EOF: Incomplete - -def is_EOF(value: Any) -> bool: - """Checks if a value is an EOF signal.""" - -class InputType(Enum): - """Input type contains all input types supported by Flyde.""" - DYNAMIC = 'dynamic' - NUMBER = 'number' - BOOLEAN = 'boolean' - JSON = 'json' - STRING = 'string' - -@dataclass -class InputConfig: - """Configuration of an input in a Flyde flow.""" - type: InputType - value: Any | None = ... - def __init__(self, type, value=...) -> None: ... - -class InputMode(Enum): - """InputMode is the mode of an input. - - QUEUE: The input is connected to a queue. On each node invocation, a new value is taken from the queue. - If the queue is empty, the node invocation is blocked. - STICKY: The input has a sticky value. It has a queue attached to it, but the last received value is returned in - absence of new values in the queue. Thus sticky inputs are non-blocking. - STATIC: The input has a static value that does not change.""" - QUEUE = 'queue' - STICKY = 'sticky' - STATIC = 'static' - -class Requiredness(Enum): - """Requiredness of an input. - - REQUIRED: The input is required to be connected. - OPTIONAL: The input is optional. - REQUIRED_IF_CONNECTED: The input is required if it is connected to a queue.""" - REQUIRED = 'required' - OPTIONAL = 'optional' - REQUIRED_IF_CONNECTED = 'required-if-connected' - -class OutputMode(Enum): - """OutputMode defines the behavior of an output if it is connected to multiple input queues. - - REF: Copy-by-reference. Each connected input will receive the same object. - VALUE: Copy-by-value. Each connected input will receive a deep copy of the object. - CIRCLE: Circular. Each connected input will receive the object in a round-robin fashion. - """ - REF = 'ref' - VALUE = 'value' - CIRCLE = 'circle' - -class Input: - """Input is an interface for getting input/output data for a node.""" - id: Incomplete - description: Incomplete - type: Incomplete - _input_mode: Incomplete - _value: Incomplete - required: Incomplete - _ref_count: int - def __init__(self, /, id: str = '', description: str = '', mode: InputMode = ..., type: type | None = None, value: Any = None, required: Requiredness = ...) -> None: - """Create a new input object. - - Args: - id (str): The ID of the input - description (str): The description of the input - mode (InputMode): The mode of the input - typ (type): The type of the input - value (Any): The value of the input for InputMode = InputMode.STATIC or InputMode = InputMode.STICKY - required (Required): The requiredness of the input - """ - _queue: Incomplete - @property - def queue(self) -> Queue: - """Get the queue of the input.""" - @property - def is_connected(self) -> bool: - """Check if the input is connected to a queue.""" - @property - def value(self) -> Any: - """Get the static value associated with the input.""" - @value.setter - def value(self, value: Any): - """Set the static value of the input.""" - def get(self) -> Any: - """Get the value of the input from either the queue or static value.""" - def empty(self) -> bool: - """Check if the input queue is empty.""" - def count(self) -> int: - """Get the number of elements in the input queue.""" - def inc_ref_count(self) -> None: - """Increment the reference count of the input.""" - def dec_ref_count(self) -> None: - """Decrement the reference count of the input.""" - @property - def ref_count(self) -> int: - """Get the reference count of the input.""" - def apply_config(self, config: InputConfig): - """Apply config from the flyde flow to the input.""" - -class Output: - """Output is an interface for setting output data for a component.""" - id: Incomplete - description: Incomplete - _output_mode: Incomplete - type: Incomplete - delayed: Incomplete - _queues: Incomplete - _circle_index: int - def __init__(self, /, id: str = '', description: str = '', mode: OutputMode = ..., type: type | None = None, delayed: bool = False) -> None: - """Create a new output object. - - Args: - id (str): The ID of the output - description (str): The description of the output - type (type): The type of the output - delayed (bool): If the output is delayed [not implemented yet] - """ - def connect(self, queue: Queue): - """Connect a queue to the output. - - This method can be called multiple times to connect multiple queues to the same output. - """ - @property - def connected(self) -> bool: - """Check if the output is connected to a queue.""" - def send(self, value: Any): - """Put a value in the output queue.""" - -class RedirectQueue: - """RedriveQueue is a fake write-only queue that is used by GraphPort - to redrive input values to the output queues.""" - _output: Incomplete - _ref_count: int - def __init__(self, output: Output) -> None: ... - @property - def ref_count(self) -> int: ... - def inc_ref_count(self) -> None: ... - def dec_ref_count(self) -> None: ... - def put(self, item: Any, block: bool = True, timeout: Incomplete | None = None): ... - -class GraphPort(Input, Output): - """GraphPort is an interface between inside and outside of the graph used for input/output. - - It combines Input and Output, because Graph Input acts as an Input for outside world, - but outputs values inside the graph. Similarly, Graph Output acts as an Output for outside world, - but receives values from inside the graph.""" - _queue: Incomplete - def __init__(self, id: str = '', description: str = '', type: type | None = None, value: Any = None, required: Requiredness = ..., output_mode: OutputMode = ..., delayed: bool = False) -> None: ... - def inc_ref_count(self): ... - def dec_ref_count(self): ... - -class ConnectionNode: - """ConnectionNode is a combination of a node and an input/output pin. - - It is used as a source or destination of a connection.""" - ins_id: Incomplete - pin_id: Incomplete - def __init__(self, ins_id: str, pin_id: str) -> None: ... - -class Connection: - """Connection is a connection between two nodes in a graph.""" - from_node: Incomplete - to_node: Incomplete - delayed: Incomplete - hidden: Incomplete - def __init__(self, from_node: ConnectionNode, to_node: ConnectionNode, delayed: bool = False, hidden: bool = False) -> None: ... - @classmethod - def from_yaml(cls, yml: dict): - """Create a connection from a parsed YAML dictionary.""" - def to_dict(self) -> dict: ... diff --git a/flyde/node.pyi b/flyde/node.pyi deleted file mode 100644 index 2f1968b..0000000 --- a/flyde/node.pyi +++ /dev/null @@ -1,139 +0,0 @@ -import abc -from _typeshed import Incomplete -from abc import ABC, abstractmethod -from dataclasses import dataclass -from enum import Enum -from flyde.io import Connection as Connection, EOF as EOF, GraphPort as GraphPort, Input as Input, InputConfig as InputConfig, InputMode as InputMode, InputType as InputType, Output as Output, Requiredness as Requiredness, is_EOF as is_EOF -from threading import Event -from typing import Any, Callable - -logger: Incomplete -SUPPORTED_MACROS: Incomplete - -class InstanceType(Enum): - """InstanceType is the type of an instance. - - VISUAL: The instance is a visual node. - CODE: The instance is a code node. - """ - VISUAL = 'visual' - CODE = 'code' - -class InstanceSourceType(Enum): - """InstanceSourceType is the source type of an instance. - - FILE: The instance is created from a file. - PACKAGE: The instance is created from a built in package. - CUSTOM: The instance is created from a custom module with path format.""" - FILE = 'file' - PACKAGE = 'package' - CUSTOM = 'custom' - -@dataclass -class InstanceSource: - """Source configuration of an instance.""" - type: InstanceSourceType - data: str - def __init__(self, type, data) -> None: ... - -@dataclass -class InstanceArgs: - """Arguments to pass to the instance factory.""" - id: str - display_name: str - stopped: Event | None - config: dict[str, Any] - type: InstanceType = ... - source: InstanceSource | None = ... - def to_dict(self) -> dict: - """Convert the instance arguments to a dictionary.""" - def __init__(self, id, display_name, stopped, config, type=..., source=...) -> None: ... -InstanceFactory = Callable[[str, InstanceArgs], Any] - -class Node(ABC, metaclass=abc.ABCMeta): - """Node is the main building block of an application. - - Attributes: - id (str): A unique identifier for the node. - node_type (str): The node type identifier. - config (dict): A dictionary of input pin configurations. - display_name (str): A human-readable name for the node. - inputs (dict[str, Input]): Node input map. - outputs (dict[str, Output]): Node output map. - """ - inputs: dict[str, Input] - outputs: dict[str, Output] - _node_type: Incomplete - _id: Incomplete - _display_name: Incomplete - _config_raw: Incomplete - _config: Incomplete - _stopped: Incomplete - def __init__(self, /, id: str, node_type: str = '', display_name: str = '', inputs: dict[str, Input] = {}, outputs: dict[str, Output] = {}, stopped: Event = ..., config: dict[str, InputConfig] = {}) -> None: ... - def parse_config(self, config: dict[str, Any]) -> dict[str, Any]: - """Parse the raw config into a typed config dictionary.""" - @abstractmethod - def run(self): - """Run the node. This method should be overridden by subclasses.""" - @abstractmethod - def stop(self): - """Stop the node. This method should be overridden by subclasses.""" - def finish(self) -> None: - """Finish the component execution gracefully by closing all its outputs and notifying others.""" - @property - def stopped(self) -> Event: ... - def shutdown(self) -> None: - """Shutdown the component. This method is optional and can be overridden by subclasses.""" - def send(self, output_id: str, value: Any): - """Send a value to an output.""" - def receive(self, input_id: str) -> Any: - """Receive a value from an input.""" - @classmethod - def from_yaml(cls, create: InstanceFactory, yml: dict): - """Create a node from a parsed YAML dictionary.""" - def to_dict(self) -> dict: ... - -class Component(Node): - """A node that runs a function when executed.""" - _stop: Incomplete - _mutex: Incomplete - def __init__(self, **kwargs) -> None: ... - def run(self) -> None: ... - def stop(self) -> None: - """Stop the component execution.""" - -class Graph(Node): - """A visual graph node that contains other nodes.""" - inputs: Incomplete - outputs: Incomplete - _connections: Incomplete - _instances: Incomplete - _instances_stopped: Incomplete - def __init__(self, /, id: str = '', node_type: str = '', config: dict[str, InputConfig] = {}, display_name: str = '', instances: dict[str, Node] = {}, instances_stopped: dict[str, Event] = {}, connections: list[Connection] = [], inputs: dict[str, GraphPort] = {}, outputs: dict[str, GraphPort] = {}, stopped: Event = ...) -> None: ... - def _check_pin(self, pin_type: str, instance_id: str, pin_id: str): - """Check if the instance and pin exist.""" - def run(self) -> None: - """Run the graph.""" - def shutdown(self) -> None: - """Call shutdown handlers on all instances. - - This method is called from the main thread to allow cleanup and things like UI.""" - def stop(self) -> None: - """Stop all instances gracefully.""" - def terminate(self) -> None: - """Terminate all instances immediately.""" - @property - def stopped(self) -> Event: - """Return the stopped event which is set when the node has stopped.""" - _stopped: Incomplete - @stopped.setter - def stopped(self, value: Event): - """Set the stopped event.""" - @classmethod - def from_yaml(cls, create: InstanceFactory, yml: dict): - """Create a Graph node from a parsed YAML dictionary.""" - def to_dict(self) -> dict: - """Return a dictionary representation of the node.""" - -def create_instance_id(node_type: str) -> str: - """Create a unique instance ID.""" diff --git a/flyde/nodes.pyi b/flyde/nodes.pyi deleted file mode 100644 index 88eb9c7..0000000 --- a/flyde/nodes.pyi +++ /dev/null @@ -1,61 +0,0 @@ -from _typeshed import Incomplete -from dataclasses import dataclass -from enum import Enum -from flyde.io import Input as Input, InputConfig as InputConfig, InputMode as InputMode, InputType as InputType, Output as Output, Requiredness as Requiredness -from flyde.node import Component as Component -from typing import Any - -class InlineValue(Component): - """InlineValue sends a constant value to output.""" - outputs: Incomplete - value: Incomplete - def __init__(self, **kwargs) -> None: ... - def process(self) -> None: ... - -class _ConditionType(Enum): - """Condition type enumeration.""" - Equal = 'EQUAL' - NotEqual = 'NOT_EQUAL' - Contains = 'CONTAINS' - NotContains = 'NOT_CONTAINS' - RegexMatches = 'REGEX_MATCHES' - Exists = 'EXISTS' - NotExists = 'NOT_EXISTS' - -@dataclass -class _ConditionConfig: - """Configuration etry for the condition type.""" - type: _ConditionType - def __init__(self, type) -> None: ... - -class _ConditionalConfig: - """Conditional configuration.""" - condition_type: Incomplete - left_operand: Incomplete - right_operand: Incomplete - def __init__(self, config: dict[str, InputConfig | _ConditionConfig]) -> None: ... - -class Conditional(Component): - """Conditional component evaluates a condition against the input and sends the result to output.""" - inputs: Incomplete - outputs: Incomplete - def parse_config(self, config: dict[str, Any]) -> dict[str, Any]: - """Parse the raw config, handling the 'condition' special case.""" - _config: Incomplete - def __init__(self, **kwargs) -> None: ... - def _evaluate(self, left_operand: Any, right_operand: Any) -> bool: ... - def process(self, leftOperand: Any, rightOperand: Any): ... - -class GetAttribute(Component): - """Get an attribute from an object or dictionary.""" - inputs: Incomplete - outputs: Incomplete - def __init__(self, **kwargs) -> None: ... - def process(self, object: Any, key: str): ... - -class Http(Component): - """Http component makes HTTP requests with urllib.""" - inputs: Incomplete - outputs: Incomplete - def __init__(self, **kwargs) -> None: ... - def process(self, url: str, method: str, headers: dict | None = None, params: dict | None = None, data: dict | None = None): ... diff --git a/flyde/__init__.pyi b/flyde/py.typed similarity index 100% rename from flyde/__init__.pyi rename to flyde/py.typed diff --git a/pyproject.toml b/pyproject.toml index a7e022e..bdfdc8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,9 @@ pyflyde = "flyde.cli:main" # package-dir = { "" = "flyde" } packages = ["flyde"] +[tool.setuptools.package-data] +flyde = ["py.typed"] + [project.optional-dependencies] dev = [ "setuptools", diff --git a/tests/test_cli.py b/tests/test_cli.py index 7a43f0f..e89dcc2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -94,6 +94,7 @@ class TestGenerateNodeJson(unittest.TestCase): def test_generate_custom_node_json(self): result = generate_node_json("CustomBob", CustomBob, "test_components.py") + assert isinstance(result, dict) expected = { "id": "CustomBob", "type": "code", @@ -129,6 +130,7 @@ class NodeWithoutDoc(Component): result = generate_node_json("NodeWithoutDoc", NodeWithoutDoc, "test.py") + assert isinstance(result, dict) self.assertEqual(result["description"], "") self.assertEqual(result["editorNode"]["description"], "") @@ -138,6 +140,7 @@ class EmptyNode(Component): result = generate_node_json("EmptyNode", EmptyNode, "test.py") + assert isinstance(result, dict) self.assertEqual(result["editorNode"]["inputs"], {}) self.assertEqual(result["editorNode"]["outputs"], {}) From 9aa5a20784734debc7f374137529721f630d60c8 Mon Sep 17 00:00:00 2001 From: Vladimir Sibirov Date: Sat, 26 Jul 2025 19:50:07 +0200 Subject: [PATCH 17/18] Deprecate SUPPORTED_MACROS --- flyde/cli.py | 5 ++-- flyde/node.py | 2 -- flyde/nodes.py | 62 +++++++++++++++++++++++------------------------ tests/test_cli.py | 17 +++++++------ 4 files changed, 43 insertions(+), 43 deletions(-) diff --git a/flyde/cli.py b/flyde/cli.py index c758a53..a372d1d 100644 --- a/flyde/cli.py +++ b/flyde/cli.py @@ -14,7 +14,8 @@ import yaml from flyde.flow import Flow, add_folder_to_path -from flyde.node import SUPPORTED_MACROS, Component +from flyde.node import Component +from flyde.nodes import list_nodes log_level = getattr(logging, os.getenv("LOG_LEVEL", "INFO").upper(), logging.INFO) logging.basicConfig(level=log_level) @@ -53,7 +54,7 @@ def convert_class_name_to_display_name(class_name: str) -> str: def is_stdlib_node(node_name: str) -> bool: """Check if a node name matches a stdlib node.""" - return node_name in SUPPORTED_MACROS + return node_name in list_nodes() def collect_components_from_directory(directory_path: str) -> dict: diff --git a/flyde/node.py b/flyde/node.py index 403aacd..75ecc48 100644 --- a/flyde/node.py +++ b/flyde/node.py @@ -11,8 +11,6 @@ logger = logging.getLogger(__name__) -SUPPORTED_MACROS = ["InlineValue", "Conditional", "GetAttribute", "Http"] - class InstanceType(Enum): """InstanceType is the type of an instance. diff --git a/flyde/nodes.py b/flyde/nodes.py index 94cc21e..a64032d 100644 --- a/flyde/nodes.py +++ b/flyde/nodes.py @@ -1,3 +1,4 @@ +import inspect import json import re from dataclasses import dataclass @@ -9,6 +10,27 @@ from flyde.node import Component +def list_nodes(): + """Dynamically discover all Component classes defined in this module. + + Returns: + list: List of class names that inherit from Component + """ + current_module = inspect.getmodule(inspect.currentframe()) + component_classes = [] + + for name, obj in inspect.getmembers(current_module): + if ( + inspect.isclass(obj) + and issubclass(obj, Component) + and obj != Component + and obj.__module__ == current_module.__name__ + ): + component_classes.append(name) + + return sorted(component_classes) + + class InlineValue(Component): """InlineValue sends a constant value to output.""" @@ -84,11 +106,7 @@ def parse_config(self, config: dict[str, Any]) -> dict[str, Any]: result = super().parse_config(config) # type: ignore # Handle the condition special case - if ( - "condition" in result - and isinstance(result["condition"], dict) - and "type" in result["condition"] - ): + if "condition" in result and isinstance(result["condition"], dict) and "type" in result["condition"]: result["condition"] = _ConditionConfig(**result["condition"]) return result @@ -96,16 +114,10 @@ def parse_config(self, config: dict[str, Any]) -> dict[str, Any]: def __init__(self, **kwargs): super().__init__(**kwargs) self._config = _ConditionalConfig(self._config) - if ( - hasattr(self._config, "left_operand") - and self._config.left_operand.type != InputType.DYNAMIC - ): + if hasattr(self._config, "left_operand") and self._config.left_operand.type != InputType.DYNAMIC: self.inputs["leftOperand"]._input_mode = InputMode.STATIC self.inputs["leftOperand"].value = self._config.left_operand.value - if ( - hasattr(self._config, "right_operand") - and self._config.right_operand.type != InputType.DYNAMIC - ): + if hasattr(self._config, "right_operand") and self._config.right_operand.type != InputType.DYNAMIC: self.inputs["rightOperand"]._input_mode = InputMode.STATIC self.inputs["rightOperand"].value = self._config.right_operand.value @@ -123,9 +135,7 @@ def _evaluate(self, left_operand: Any, right_operand: Any) -> bool: m = re.match(right_operand, left_operand) return m is not None elif condition_type == _ConditionType.Exists: - return ( - left_operand is not None and left_operand != "" and left_operand != [] - ) + return left_operand is not None and left_operand != "" and left_operand != [] elif condition_type == _ConditionType.NotExists: return left_operand is None or left_operand == "" or left_operand == [] else: @@ -185,18 +195,10 @@ class Http(Component): icon = "globe" inputs = { "url": Input(description="URL to request", required=Requiredness.REQUIRED), - "method": Input( - description="HTTP method", type=str, required=Requiredness.REQUIRED - ), - "headers": Input( - description="HTTP headers", type=dict, required=Requiredness.OPTIONAL - ), - "params": Input( - description="URL parameters", type=dict, required=Requiredness.OPTIONAL - ), - "data": Input( - description="Request body", type=dict, required=Requiredness.OPTIONAL - ), + "method": Input(description="HTTP method", type=str, required=Requiredness.REQUIRED), + "headers": Input(description="HTTP headers", type=dict, required=Requiredness.OPTIONAL), + "params": Input(description="URL parameters", type=dict, required=Requiredness.OPTIONAL), + "data": Input(description="Request body", type=dict, required=Requiredness.OPTIONAL), } outputs = { "data": Output(description="Response data"), @@ -222,9 +224,7 @@ def __init__(self, **kwargs): self.inputs["url"]._input_mode = InputMode.STATIC self.inputs["url"].value = self._config["url"].value - if "headers" in self._config and isinstance( - self._config["headers"], InputConfig - ): + if "headers" in self._config and isinstance(self._config["headers"], InputConfig): if self._config["headers"].type == InputType.DYNAMIC: self.inputs["headers"]._input_mode = InputMode.STICKY else: diff --git a/tests/test_cli.py b/tests/test_cli.py index e89dcc2..6ca1b3a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -14,7 +14,8 @@ is_stdlib_node, ) from flyde.io import Input, Output -from flyde.node import SUPPORTED_MACROS, Component +from flyde.node import Component +from flyde.nodes import list_nodes class TestCLIHelpers(unittest.TestCase): @@ -37,7 +38,7 @@ def test_convert_class_name_to_display_name(self): def test_is_stdlib_node(self): # Test that all supported macros are detected as stdlib nodes - for macro in SUPPORTED_MACROS: + for macro in list_nodes(): with self.subTest(node_name=macro): result = is_stdlib_node(macro) self.assertTrue(result) @@ -440,11 +441,11 @@ class CustomAlice(Component): # Check nodes nodes = data["nodes"] # Should have all custom nodes plus all stdlib nodes - expected_nodes = set(["CustomBob", "CustomAlice"] + list(SUPPORTED_MACROS)) + expected_nodes = set(["CustomBob", "CustomAlice"] + list_nodes()) self.assertEqual(set(nodes.keys()), expected_nodes) self.assertIn("CustomBob", nodes) self.assertIn("CustomAlice", nodes) - for stdlib_node in SUPPORTED_MACROS: + for stdlib_node in list_nodes(): self.assertIn(stdlib_node, nodes) # Check CustomBob @@ -468,7 +469,7 @@ class CustomAlice(Component): if custom_group is None or stdlib_group is None: return self.assertCountEqual(custom_group["nodeIds"], ["CustomBob", "CustomAlice"]) - self.assertCountEqual(stdlib_group["nodeIds"], list(SUPPORTED_MACROS)) + self.assertCountEqual(stdlib_group["nodeIds"], list_nodes()) def test_gen_json_with_flyde_files(self): # Create test files with both .py components and copy an existing .flyde file @@ -507,7 +508,7 @@ class CustomNode(Component): # Check nodes - should have both Python and .flyde nodes nodes = data["nodes"] - expected_custom_nodes = set(["CustomNode", "TestFlow"] + list(SUPPORTED_MACROS)) + expected_custom_nodes = set(["CustomNode", "TestFlow"] + list_nodes()) self.assertEqual(set(nodes.keys()), expected_custom_nodes) # Check CustomNode (Python) @@ -548,7 +549,7 @@ def test_gen_json_only_flyde_files(self): # Should have .flyde nodes and stdlib nodes nodes = data["nodes"] - expected_nodes = set(["Flow1", "Flow2"] + list(SUPPORTED_MACROS)) + expected_nodes = set(["Flow1", "Flow2"] + list_nodes()) self.assertEqual(set(nodes.keys()), expected_nodes) # Check that .flyde nodes are in the custom group @@ -655,4 +656,4 @@ class Conditional(Component): self.assertIsNotNone(stdlib_group) if stdlib_group is None: return - self.assertCountEqual(stdlib_group["nodeIds"], list(SUPPORTED_MACROS)) + self.assertCountEqual(stdlib_group["nodeIds"], list_nodes()) From c8d638bfd80eea288d7760e71f290487c29369fe Mon Sep 17 00:00:00 2001 From: Vladimir Sibirov Date: Sat, 26 Jul 2025 19:50:59 +0200 Subject: [PATCH 18/18] Bump version to 0.1.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bdfdc8e..72ef2c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pyflyde" -version = "0.1.0-alpha" +version = "0.1.0" requires-python = ">= 3.9" authors = [{ name = "Vladimir Sibirov" }] description = "Python SDK and runtime for Flyde - a visual flow-based programming language and IDE."