diff --git a/Makefile b/Makefile index 5bfeffb..1ad9ea9 100644 --- a/Makefile +++ b/Makefile @@ -7,25 +7,18 @@ SRC_DIR = flyde TEST_DIR = tests # Targets -.PHONY: ts test cover +.PHONY: gen test cover -ts: - @echo "Building the project..." - # For each *.py file in the examples/mylib directory run the ./flyde.py gen command to generate TS bindings - @for file in $(LIB_DIR)/*.py; do \ - ./flyde.py gen $$file; \ - done +gen: + @echo "Generating component definitions..." + # Generate JSON definitions for the examples/mylib directory + @./pyflyde gen $(LIB_DIR)/ lint: @echo "Running linters..." @black $(LIB_DIR) $(TEST_DIR); @flake8 $(LIB_DIR) $(TEST_DIR); -stubgen: - @echo "Generating type stubs..." - @rm -f $(SRC_DIR)/*.pyi; - @stubgen $(SRC_DIR) --include-docstrings --include-private -o .; - test: @echo "Running tests..." @$(PYTHON) -m unittest discover -s $(TEST_DIR) -p "test_$(if $(mod),$(mod),*).py"; @@ -46,7 +39,7 @@ builddist: @rm -f ./dist/* @$(PYTHON) -m build; -release: lint test stubgen builddist +release: lint test gen builddist @echo "Releasing the project..."; upload: diff --git a/README.md b/README.md index df07c55..edcfe00 100644 --- a/README.md +++ b/README.md @@ -61,13 +61,13 @@ Install Flyde VSCode extension from the [marketplace](https://marketplace.visual You can browse the component library in the panel on the right. To see your local components click the "View all" button. They will appear under the "Current project". Note that PyFlyde doesn't implement all of the Flyde's stdlib components, only a few essential ones. -Whenever you change your component library classes or their interfaces, use `pyflyde gen` command to generate `.flyde.ts` definitions, e.g.: +Whenever you change your component library classes or their interfaces, use `pyflyde gen` command to generate `flyde-nodes.json` definitions, e.g.: ```bash -pyflyde gen examples/mylib/components.py +pyflyde gen examples/ ``` -Flyde editor needs `.flyde.ts` files in order to "see" your components. +This will recursively scan all Python files in the directory and its subdirectories to find PyFlyde components and generate a `flyde-nodes.json` file with relative paths. Flyde editor needs `flyde-nodes.json` files in order to "see" your components. ### Running a Machine Learning example and creating your first project diff --git a/docs/quickstart.md b/docs/quickstart.md index b08b611..a5cd726 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -37,12 +37,20 @@ where = ["mylib"] # With this line Install the dependencies: ```bash -pip install examples/ +pip install examples/ ``` ## Running the Hello World example -Run the example flow: +First, generate the component metadata for the examples: + +```bash +pyflyde gen examples/ +``` + +This will recursively scan all Python files in the `examples/` directory and generate a `flyde-nodes.json` file with metadata for all PyFlyde components found. + +Then run the example flow: ```bash pyflyde examples/HelloPy.flyde @@ -54,6 +62,12 @@ It should print "Hello Flyde!" in the console. `examples/Clustering.flyde` is a more complex example which uses Pandas and Scikit-Learn to run K-means clustering on a [wine clustering dataset from Kaggle](https://www.kaggle.com/harrywang/wine-dataset-for-clustering). It's a PyFlyde version of https://github.com/Shivangi0503/Wine_Clustering_KMeans. +The component metadata should already be generated from the previous step, but if you add new components, remember to run: + +```bash +pyflyde gen examples/ +``` + Open the `examples/Clustering.flyde` in Flyde VSCode visual editor to see how it looks like. To run this example, use the `pyflyde` command line tool: diff --git a/docs/usage.md b/docs/usage.md index e9c3fc6..475a8ac 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -14,16 +14,24 @@ You can omit the `run` command because it is the default one. The following comm pyflyde examples/HelloWorld.flyde ``` -## Generating TS definitions for Flyde visual editor +## Generating component definitions for Flyde visual editor -Flyde visual editor is written for TypeScript runtime and is not aware of your Python nodes. To make your local nodes appear in the Flyde editor, you need to generate `.flyde.ts` files for them. +To make your Python nodes appear in the Flyde visual editor, you need to generate `flyde-nodes.json` metadata files for them. -For example: +Generate JSON definitions for a directory: ```bash -pyflyde gen mypackage/supermodule.py +pyflyde gen mypackage/ ``` -will generate `mypackage/supermodule.flyde.ts` TypeScript defintions for Flyde using the contents of your `mypackage.submodule` module. +This will recursively scan all `.py` files in the directory and its subdirectories, then generate a `flyde-nodes.json` file in the specified directory containing metadata for all PyFlyde components found. The paths in the generated JSON file are relative to the directory containing the `flyde-nodes.json` file, making the component library portable. -You should run `pyflyde gen` every time your create new modules containing PyFlyde nodes or whenever you update node signature (name, description, inputs, outputs, etc.). +For example, if you have components in: +- `mypackage/components.py` +- `mypackage/utils/helpers.py` + +The generated `flyde-nodes.json` will reference them as: +- `custom://components.py/ComponentName` +- `custom://utils/helpers.py/HelperComponentName` + +You should run `pyflyde gen` every time you create new modules containing PyFlyde nodes or whenever you update node signatures (name, description, inputs, outputs, etc.). diff --git a/examples/Clustering.flyde b/examples/Clustering.flyde index b6129ec..12c928a 100644 --- a/examples/Clustering.flyde +++ b/examples/Clustering.flyde @@ -1,31 +1,27 @@ -imports: - "@flyde/stdlib": - - InlineValue - - GetAttribute - mylib/dataframe.flyde.ts: - - LoadDataset - - Scale - mylib/kmeans.flyde.ts: - - KMeansNClusters - - KMeansCluster - - PCA2 - - Visualize +imports: {} node: instances: - pos: - x: -44.67793457031249 - y: -40.51885223388672 + x: -688.3592700195312 + y: 60.69355010986328 id: LoadDataset-1a039uk inputConfig: {} nodeId: LoadDataset + config: + file_path: + type: dynamic + value: "{{file_path}}" + type: code + source: + type: custom + data: custom://mylib/dataframe.py/LoadDataset - pos: - x: -81.67130859374998 - y: -124.01171112060547 + x: -939.2435131835937 + y: 61.415870666503906 id: mc4t4fqqezd1gns4fb7ckxmc inputConfig: {} - nodeId: InlineValue__mc4t4fqqezd1gns4fb7ckxmc - macroId: InlineValue - macroData: + nodeId: InlineValue + config: type: type: string value: string @@ -35,32 +31,65 @@ node: label: type: string value: wine-clustering.csv + type: code + source: + type: package + data: "@flyde/nodes" - pos: - x: -21.232984619140666 - y: 59.69538116455078 + x: -436.9881726074219 + y: 60.95093536376953 id: Scale-dz139az inputConfig: {} nodeId: Scale + config: + dataframe: + type: dynamic + value: "{{dataframe}}" + type: code + source: + type: custom + data: custom://mylib/dataframe.py/Scale - pos: - x: 115.65844161987303 - y: 212.51250839233398 + x: -24.11493484497072 + y: 62.353389739990234 id: KMeansNClusters-5m2390u inputConfig: {} nodeId: KMeansNClusters + config: + scaled_dataframe: + type: dynamic + value: "{{scaled_dataframe}}" + max_clusters: + type: dynamic + value: "{{max_clusters}}" + type: code + source: + type: custom + data: custom://mylib/kmeans.py/KMeansNClusters - pos: - x: -21.067138671875 - y: 348.4979133605957 + x: 234.1243896484375 + y: 185.80514907836914 id: KMeansCluster-ql339i4 inputConfig: {} nodeId: KMeansCluster + config: + scaled_dataframe: + type: dynamic + value: "{{scaled_dataframe}}" + n_clusters: + type: dynamic + value: "{{n_clusters}}" + type: code + source: + type: custom + data: custom://mylib/kmeans.py/KMeansCluster - pos: - x: 233.42520244598387 - y: 80.12830471992493 + x: -431.2221974563599 + y: -46.37674593925476 id: yquy5xqil6xp9tmb42ktmosp inputConfig: {} - nodeId: InlineValue__yquy5xqil6xp9tmb42ktmosp - macroId: InlineValue - macroData: + nodeId: InlineValue + config: value: type: number value: 20 @@ -70,42 +99,81 @@ node: label: type: string value: "20" + type: code + source: + type: package + data: "@flyde/nodes" - pos: - x: -264.7014599609375 - y: 436.35204895019535 + x: 234.12208740234377 + y: 410.2254124450684 id: PCA2-mz439jk inputConfig: {} nodeId: PCA2 + config: + scaled_dataframe: + type: dynamic + value: "{{scaled_dataframe}}" + type: code + source: + type: custom + data: custom://mylib/kmeans.py/PCA2 - pos: - x: -194.03480346679686 - y: 829.8976345825196 + x: 1010.514711303711 + y: 252.2306843566895 id: Visualize-yz539m4 inputConfig: {} nodeId: Visualize + config: + pca_components: + type: dynamic + value: "{{pca_components}}" + pca_centroids: + type: dynamic + value: "{{pca_centroids}}" + kmeans_result: + type: dynamic + value: "{{kmeans_result}}" + type: code + source: + type: custom + data: custom://mylib/kmeans.py/Visualize - pos: - x: -85.56712158203123 - y: 675.0107851791382 + x: 736.7720812988282 + y: 149.70720727920536 id: PCA2-e5639ct inputConfig: {} nodeId: PCA2 + config: + scaled_dataframe: + type: dynamic + value: "{{scaled_dataframe}}" + type: code + source: + type: custom + data: custom://mylib/kmeans.py/PCA2 - pos: - x: -158.16623901367188 - y: 567.0884128027336 + x: 513.1246850585937 + y: 141.11043981841965 id: nmt2k6i80qpwu9fyuckbqefw inputConfig: {} - nodeId: GetAttribute__nmt2k6i80qpwu9fyuckbqefw - macroId: GetAttribute - macroData: + nodeId: GetAttribute + config: key: type: dynamic + object: + type: dynamic + value: "{{object}}" + type: code + source: + type: package + data: "@flyde/nodes" - pos: - x: -94.66524780273448 - y: 458.4299138098137 + x: 234.12473022460927 + y: 310.8023236303703 id: xb5xo767x6u0ubdvvegi0e53 inputConfig: {} - nodeId: InlineValue__xb5xo767x6u0ubdvvegi0e53 - macroId: InlineValue - macroData: + nodeId: InlineValue + config: type: type: string value: string @@ -115,6 +183,10 @@ node: label: type: string value: '"centroids"' + type: code + source: + type: package + data: "@flyde/nodes" connections: - from: insId: mc4t4fqqezd1gns4fb7ckxmc diff --git a/examples/HelloPy.flyde b/examples/HelloPy.flyde index 350eae9..802cc67 100644 --- a/examples/HelloPy.flyde +++ b/examples/HelloPy.flyde @@ -1,34 +1,26 @@ -imports: - "@flyde/stdlib": - - InlineValue - mylib/components.flyde.ts: - - Print +imports: {} node: instances: - pos: - x: -157.29365234375 - y: -130.58668701171877 + x: -156.29365234375 + y: -256.58668701171877 id: Print-g7039qo - inputConfig: {} + inputConfig: + __trigger: + mode: queue + msg: + mode: sticky + visibleInputs: [] nodeId: Print - - pos: - x: -198.96331298828125 - y: -295.8833435058594 - id: vzy5s9fyaacybwutaijgttv1 - inputConfig: {} - nodeId: InlineValue__vzy5s9fyaacybwutaijgttv1 - macroId: InlineValue - macroData: - value: + config: + msg: type: string - value: Hello Flyde! - connections: - - from: - insId: vzy5s9fyaacybwutaijgttv1 - pinId: value - to: - insId: Print-g7039qo - pinId: msg + value: Hello, Flyde! + type: code + source: + type: custom + data: custom://mylib/components.py/Print + connections: [] id: Example inputs: {} outputs: {} diff --git a/examples/flyde-nodes.json b/examples/flyde-nodes.json new file mode 100644 index 0000000..926943d --- /dev/null +++ b/examples/flyde-nodes.json @@ -0,0 +1,325 @@ +{ + "nodes": { + "LoadDataset": { + "id": "LoadDataset", + "type": "code", + "displayName": "Load Dataset", + "description": "Loads a dataset from a file into a DataFrame.", + "icon": "fa-brands fa-python", + "source": { + "type": "custom", + "data": "custom://mylib/dataframe.py/LoadDataset" + }, + "editorNode": { + "id": "LoadDataset", + "displayName": "Load Dataset", + "description": "Loads a dataset from a file into a DataFrame.", + "inputs": { + "file_path": { + "description": "The path to the file containing the dataset" + } + }, + "outputs": { + "dataframe": { + "description": "The loaded dataframe" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "Scale": { + "id": "Scale", + "type": "code", + "displayName": "Scale", + "description": "Scales the features of a dataframe with a scikit-learn StandardScaler.", + "icon": "fa-brands fa-python", + "source": { + "type": "custom", + "data": "custom://mylib/dataframe.py/Scale" + }, + "editorNode": { + "id": "Scale", + "displayName": "Scale", + "description": "Scales the features of a dataframe with a scikit-learn StandardScaler.", + "inputs": { + "dataframe": { + "description": "The dataframe to scale" + } + }, + "outputs": { + "scaled_dataframe": { + "description": "The scaled dataframe" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "KMeansCluster": { + "id": "KMeansCluster", + "type": "code", + "displayName": "KMeans Cluster", + "description": "Clusters the dataframe using K-means clustering.", + "icon": "fa-brands fa-python", + "source": { + "type": "custom", + "data": "custom://mylib/kmeans.py/KMeansCluster" + }, + "editorNode": { + "id": "KMeansCluster", + "displayName": "KMeans Cluster", + "description": "Clusters the dataframe using K-means clustering.", + "inputs": { + "scaled_dataframe": { + "description": "The scaled dataframe to cluster" + }, + "n_clusters": { + "description": "The number of clusters" + } + }, + "outputs": { + "kmeans_result": { + "description": "K-means clustering result" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "KMeansNClusters": { + "id": "KMeansNClusters", + "type": "code", + "displayName": "KMeans NClusters", + "description": "Finds the optimal number of clusters for K-means clustering using silhouette method.", + "icon": "fa-brands fa-python", + "source": { + "type": "custom", + "data": "custom://mylib/kmeans.py/KMeansNClusters" + }, + "editorNode": { + "id": "KMeansNClusters", + "displayName": "KMeans NClusters", + "description": "Finds the optimal number of clusters for K-means clustering using silhouette method.", + "inputs": { + "scaled_dataframe": { + "description": "The scaled dataframe to cluster" + }, + "max_clusters": { + "description": "The maximum number of clusters to consider" + } + }, + "outputs": { + "n_clusters": { + "description": "The optimal number of clusters" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "PCA2": { + "id": "PCA2", + "type": "code", + "displayName": "PCA2", + "description": "Performs PCA on a dataframe and returns the first two principal components.", + "icon": "fa-brands fa-python", + "source": { + "type": "custom", + "data": "custom://mylib/kmeans.py/PCA2" + }, + "editorNode": { + "id": "PCA2", + "displayName": "PCA2", + "description": "Performs PCA on a dataframe and returns the first two principal components.", + "inputs": { + "scaled_dataframe": { + "description": "The scaled dataframe to reduce" + } + }, + "outputs": { + "pca_components": { + "description": "The first two principal components" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "Visualize": { + "id": "Visualize", + "type": "code", + "displayName": "Visualize", + "description": "Visualizes the clustered dataframe.", + "icon": "fa-brands fa-python", + "source": { + "type": "custom", + "data": "custom://mylib/kmeans.py/Visualize" + }, + "editorNode": { + "id": "Visualize", + "displayName": "Visualize", + "description": "Visualizes the clustered dataframe.", + "inputs": { + "pca_components": { + "description": "The first two principal components" + }, + "pca_centroids": { + "description": "The centroids in PCA space" + }, + "kmeans_result": { + "description": "K-means clustering result" + } + }, + "outputs": {}, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "Concat": { + "id": "Concat", + "type": "code", + "displayName": "Concat", + "description": "Concatenates two strings.", + "icon": "fa-brands fa-python", + "source": { + "type": "custom", + "data": "custom://mylib/components.py/Concat" + }, + "editorNode": { + "id": "Concat", + "displayName": "Concat", + "description": "Concatenates two strings.", + "inputs": { + "a": { + "description": "The first string" + }, + "b": { + "description": "The second string" + } + }, + "outputs": { + "out": { + "description": "The concatenated string" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "Print": { + "id": "Print", + "type": "code", + "displayName": "Print", + "description": "Prints the input message to the console.", + "icon": "fa-brands fa-python", + "source": { + "type": "custom", + "data": "custom://mylib/components.py/Print" + }, + "editorNode": { + "id": "Print", + "displayName": "Print", + "description": "Prints the input message to the console.", + "inputs": { + "msg": { + "description": "The message to print" + } + }, + "outputs": {}, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "Clustering": { + "id": "Clustering", + "type": "visual", + "displayName": "Clustering", + "description": "", + "icon": "fa-diagram-project", + "source": { + "type": "file", + "data": "Clustering.flyde" + }, + "editorNode": { + "id": "Clustering", + "displayName": "Clustering", + "description": "", + "inputs": {}, + "outputs": {}, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "HelloPy": { + "id": "HelloPy", + "type": "visual", + "displayName": "Hello Py", + "description": "", + "icon": "fa-diagram-project", + "source": { + "type": "file", + "data": "HelloPy.flyde" + }, + "editorNode": { + "id": "HelloPy", + "displayName": "Hello Py", + "description": "", + "inputs": {}, + "outputs": {}, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "Conditional": "@flyde/nodes", + "GetAttribute": "@flyde/nodes", + "Http": "@flyde/nodes", + "InlineValue": "@flyde/nodes" + }, + "groups": [ + { + "title": "Your PyFlyde Nodes", + "nodeIds": [ + "LoadDataset", + "Scale", + "KMeansCluster", + "KMeansNClusters", + "PCA2", + "Visualize", + "Concat", + "Print", + "Clustering", + "HelloPy" + ] + }, + { + "title": "PyFlyde Standard Nodes", + "nodeIds": [ + "Conditional", + "GetAttribute", + "Http", + "InlineValue" + ] + } + ] +} \ No newline at end of file diff --git a/examples/mylib/components.flyde.ts b/examples/mylib/components.flyde.ts deleted file mode 100644 index 20b73a8..0000000 --- a/examples/mylib/components.flyde.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { CodeNode } from "@flyde/core"; - -export const Print: CodeNode = { - id: "Print", - description: "Prints the input message to the console.", - inputs: { - msg: {"description": "The message to print"} - }, - outputs: { }, - run: () => { return; }, -}; - -export const Concat: CodeNode = { - id: "Concat", - description: "Concatenates two strings.", - inputs: { - a: {"description": "The first string"}, - b: {"description": "The second string"} - }, - outputs: { - out: {"description": "The concatenated string"} - }, - run: () => { return; }, -}; - diff --git a/examples/mylib/dataframe.flyde.ts b/examples/mylib/dataframe.flyde.ts deleted file mode 100644 index 3670e17..0000000 --- a/examples/mylib/dataframe.flyde.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { CodeNode } from "@flyde/core"; - -export const LoadDataset: CodeNode = { - id: "LoadDataset", - description: "Loads a dataset from a file into a DataFrame.", - inputs: { - file_path: { description: "The path to the file containing the dataset" } - }, - outputs: { - dataframe: { description: "The loaded dataframe" } - }, - run: () => { return; }, -}; - -export const Scale: CodeNode = { - id: "Scale", - description: "Scales the features of a dataframe with a scikit-learn StandardScaler.", - inputs: { - dataframe: { description: "The dataframe to scale" } - }, - outputs: { - scaled_dataframe: { description: "The scaled dataframe" } - }, - run: () => { return; }, -}; - diff --git a/examples/mylib/kmeans.flyde.ts b/examples/mylib/kmeans.flyde.ts deleted file mode 100644 index 40a91f7..0000000 --- a/examples/mylib/kmeans.flyde.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { CodeNode } from "@flyde/core"; - -export const PCA2: CodeNode = { - id: "PCA2", - description: "Performs PCA on a dataframe and returns the first two principal components.", - inputs: { - scaled_dataframe: {"description": "The scaled dataframe to reduce"} - }, - outputs: { - pca_components: {"description": "The first two principal components"} - }, - run: () => { return; }, -}; - -export const KMeansNClusters: CodeNode = { - id: "KMeansNClusters", - description: "Finds the optimal number of clusters for K-means clustering using silhouette method.", - inputs: { - scaled_dataframe: {"description": "The scaled dataframe to cluster"}, - max_clusters: {"description": "The maximum number of clusters to consider"} - }, - outputs: { - n_clusters: {"description": "The optimal number of clusters"} - }, - run: () => { return; }, -}; - -export const KMeansCluster: CodeNode = { - id: "KMeansCluster", - description: "Clusters the dataframe using K-means clustering.", - inputs: { - scaled_dataframe: {"description": "The scaled dataframe to cluster"}, - n_clusters: {"description": "The number of clusters"} - }, - outputs: { - kmeans_result: {"description": "K-means clustering result"} - }, - run: () => { return; }, -}; - -export const Visualize: CodeNode = { - id: "Visualize", - description: "Visualizes the clustered dataframe.", - inputs: { - pca_components: {"description": "The first two principal components"}, - pca_centroids: {"description": "The centroids in PCA space"}, - kmeans_result: {"description": "K-means clustering result"} - }, - outputs: { }, - run: () => { return; }, -}; - diff --git a/flyde/cli.py b/flyde/cli.py index d7b8be4..a372d1d 100644 --- a/flyde/cli.py +++ b/flyde/cli.py @@ -1,37 +1,304 @@ #!/usr/bin/env python3 import argparse +import glob import importlib +import importlib.util +import json import logging import os import pprint +import re +import sys +from typing import Any, TypedDict, Union + +import yaml from flyde.flow import Flow, add_folder_to_path from flyde.node import Component +from flyde.nodes import list_nodes -log_level = getattr(logging, os.getenv('LOG_LEVEL', 'INFO').upper(), logging.INFO) +log_level = getattr(logging, os.getenv("LOG_LEVEL", "INFO").upper(), logging.INFO) logging.basicConfig(level=log_level) logger = logging.getLogger(__name__) +class EditorNodeDict(TypedDict): + id: str + displayName: str + description: str + inputs: dict[str, dict[str, str]] + outputs: dict[str, dict[str, str]] + editorConfig: dict[str, str] + + +class NodeDict(TypedDict): + id: str + type: str + displayName: str + description: str + icon: str + source: dict[str, str] + editorNode: EditorNodeDict + config: dict[str, Any] + + def py_path_to_module(py_path: str) -> str: return py_path.replace("/", ".").replace(".py", "") -def gen(path: str): - """Generate TypeScript files for a module.""" - print(f"Generating TypeScript files for module {path}") - module = py_path_to_module(path) - mod = importlib.import_module(module) - ts_file_path = path.replace(".py", ".flyde.ts") - typescript = 'import { CodeNode } from "@flyde/core";\n\n' - for name in mod.__dict__.keys(): - c = getattr(mod, name) - if name != "Component" and isinstance(c, type) and issubclass(c, Component): - typescript += c.to_ts(name) +def convert_class_name_to_display_name(class_name: str) -> str: + """Convert a class name like 'MyCustomNode' to 'My Custom Node'.""" + + return re.sub(r"(?<=[a-z])(?=[A-Z])", " ", class_name) + + +def is_stdlib_node(node_name: str) -> bool: + """Check if a node name matches a stdlib node.""" + return node_name in list_nodes() + + +def collect_components_from_directory(directory_path: str) -> dict: + """Collect all Component subclasses from .py files in a directory and its subdirectories.""" + components = {} + + # Convert directory path to absolute path + abs_dir = os.path.abspath(directory_path) + + # Add the directory to Python path for imports + if abs_dir not in sys.path: + sys.path.insert(0, abs_dir) + + # Find all .py files in the directory and subdirectories recursively + py_files = glob.glob(os.path.join(directory_path, "**", "*.py"), recursive=True) + + for py_file in py_files: + # Skip __init__.py files + if os.path.basename(py_file) == "__init__.py": + continue + + try: + # Get relative path from the directory + relative_path = os.path.relpath(py_file, directory_path) + + # Convert file path to module name (handle subdirectories) + module_path = relative_path.replace(os.path.sep, ".").replace(".py", "") + + # Import the module + spec = importlib.util.spec_from_file_location(module_path, py_file) + if spec and spec.loader: + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + # Find all Component subclasses + for name in dir(module): + obj = getattr(module, name) + if ( + name != "Component" + and isinstance(obj, type) + and issubclass(obj, Component) + and obj.__module__ == module_path + ): + components[name] = {"class": obj, "file_path": relative_path, "type": "python"} + + except Exception as e: + logger.warning(f"Failed to import module from {py_file}: {e}") + + return components + + +def collect_flyde_nodes_from_directory(directory_path: str) -> dict: + """Collect all .flyde files from a directory and its subdirectories.""" + flyde_nodes = {} + + # Find all .flyde files in the directory and subdirectories recursively + flyde_files = glob.glob(os.path.join(directory_path, "**", "*.flyde"), recursive=True) + + for flyde_file in flyde_files: + try: + # Get relative path from the directory + relative_path = os.path.relpath(flyde_file, directory_path) + + # Load the YAML content + with open(flyde_file, "r") as f: + flyde_data = yaml.safe_load(f) + + # Extract node information + node_data = flyde_data.get("node", {}) + node_id = os.path.splitext(os.path.basename(flyde_file))[0] + description = node_data.get("description", flyde_data.get("description", "")) + + # Extract inputs and outputs + inputs = node_data.get("inputs", {}) + outputs = node_data.get("outputs", {}) + + flyde_nodes[node_id] = { + "file_path": relative_path, + "description": description, + "inputs": inputs, + "outputs": outputs, + "type": "flyde", + } + + except Exception as e: + logger.warning(f"Failed to parse .flyde file {flyde_file}: {e}") - print(f"Writing TypeScript to {ts_file_path}") - with open(ts_file_path, "w") as f: - f.write(typescript) + return flyde_nodes + + +def generate_flyde_node_json(node_name: str, flyde_info: dict) -> dict: + """Generate JSON structure for a .flyde file node.""" + file_path = flyde_info["file_path"] + description = flyde_info["description"] + inputs = flyde_info["inputs"] + outputs = flyde_info["outputs"] + + display_name = convert_class_name_to_display_name(node_name) + + # Build inputs structure + editor_inputs = {} + for input_name, input_data in inputs.items(): + mode = input_data.get("mode", "required") + input_description = f"{input_name} input" + if mode == "required": + input_description += " (required)" + editor_inputs[input_name] = {"description": input_description} + + # Build outputs structure + editor_outputs = {} + for output_name, output_data in outputs.items(): + output_description = f"{output_name} output" + editor_outputs[output_name] = {"description": output_description} + + # Build the node structure + node_data = { + "id": node_name, + "type": "visual", + "displayName": display_name, + "description": description, + "icon": "fa-diagram-project", + "source": {"type": "file", "data": file_path}, + "editorNode": { + "id": node_name, + "displayName": display_name, + "description": description, + "inputs": editor_inputs, + "outputs": editor_outputs, + "editorConfig": {"type": "structured"}, + }, + "config": {}, + } + + return node_data + + +def generate_node_json(node_name: str, component_class, file_path: str = "") -> Union[dict[str, Any], str]: + """Generate JSON structure for a single component.""" + # Get node metadata + description = (component_class.__doc__ or "").strip() + display_name = convert_class_name_to_display_name(node_name) + # Use package source for stdlib nodes, custom source for others + if is_stdlib_node(node_name): + return "@flyde/nodes" + else: + source = {"type": "custom", "data": f"custom://{file_path}/{node_name}"} + icon = getattr(component_class, "icon", "fa-brands fa-python") + + # Build inputs structure + inputs = {} + if hasattr(component_class, "inputs") and component_class.inputs: + for input_name, input_obj in component_class.inputs.items(): + inputs[input_name] = {"description": input_obj.description or f"{input_name} input"} + + # Build outputs structure + outputs = {} + if hasattr(component_class, "outputs") and component_class.outputs: + for output_name, output_obj in component_class.outputs.items(): + outputs[output_name] = {"description": output_obj.description or f"{output_name} output"} + + # Build the node structure + node_data = { + "id": node_name, + "type": "code", + "displayName": display_name, + "description": description, + "icon": icon, + "source": source, + "editorNode": { + "id": node_name, + "displayName": display_name, + "description": description, + "inputs": inputs, + "outputs": outputs, + "editorConfig": {"type": "structured"}, + }, + "config": {}, + } + + return node_data + + +def gen_json(directory_path: str): + """Generate JSON file for all components in a directory.""" + print(f"Generating JSON file for components in directory {directory_path}") + + # Collect all components + components = collect_components_from_directory(directory_path) + + # Collect all .flyde nodes + flyde_nodes = collect_flyde_nodes_from_directory(directory_path) + + # Always include stdlib nodes from flyde/nodes.py + stdlib_dir = os.path.join(os.path.dirname(__file__), "nodes.py") + stdlib_components = collect_components_from_directory(os.path.dirname(stdlib_dir)) + stdlib_node_names = [name for name in stdlib_components if is_stdlib_node(name)] + + if not components and not flyde_nodes and not stdlib_node_names: + print(f"No Component subclasses or .flyde files found in directory {directory_path} or stdlib") + return + + # Build nodes structure + nodes = {} + custom_nodes = [] + stdlib_nodes = [] + + # Add user components + for node_name, component_info in components.items(): + component_class = component_info["class"] + file_path = component_info["file_path"] + nodes[node_name] = generate_node_json(node_name, component_class, file_path) + custom_nodes.append(node_name) + + # Add .flyde nodes + for node_name, flyde_info in flyde_nodes.items(): + nodes[node_name] = generate_flyde_node_json(node_name, flyde_info) + custom_nodes.append(node_name) + + # Add stdlib nodes (from flyde/nodes.py) as custom nodes, but group as stdlib overrides + for node_name in stdlib_node_names: + if node_name not in nodes: + component_class = stdlib_components[node_name]["class"] + file_path = stdlib_components[node_name]["file_path"] + nodes[node_name] = generate_node_json(node_name, component_class, file_path) + stdlib_nodes.append(node_name) + + # Build groups + groups = [] + if custom_nodes: + groups.append({"title": "Your PyFlyde Nodes", "nodeIds": custom_nodes}) + if stdlib_nodes: + groups.append({"title": "PyFlyde Standard Nodes", "nodeIds": stdlib_nodes}) + + # Build final JSON structure + json_data = {"nodes": nodes, "groups": groups} + + # Write to file + output_file = os.path.join(directory_path, "flyde-nodes.json") + with open(output_file, "w") as f: + json.dump(json_data, f, indent=2) + + print(f"Generated {output_file} with {len(nodes)} components") + print(f"Custom nodes: {custom_nodes}") + print(f"Stdlib nodes: {stdlib_nodes}") def main(): @@ -40,7 +307,7 @@ def main(): Examples: flyde.py path/to/MyFlow.flyde # Runs a flow - flyde.py gen path/to/module.py # Generates TS files for visual editor + flyde.py gen path/to/directory/ # Generates flyde-nodes.json for directory """, formatter_class=argparse.RawDescriptionHelpFormatter, ) @@ -55,7 +322,7 @@ def main(): parser.add_argument( "path", type=str, - help='Path to a ".flyde" flow file to run or a Python ".py" module to generate typescript definitions for', + help='Path to a ".flyde" flow file to run, or a directory to generate flyde-nodes.json for', ) args = parser.parse_args() @@ -75,6 +342,11 @@ def main(): add_folder_to_path(args.path) # Add current folder to path when resolving modules relative to the current folder add_folder_to_path(".") - gen(args.path) + + # Generate JSON for directory + if os.path.isdir(args.path): + gen_json(args.path) + else: + raise ValueError(f"Path {args.path} is not a directory. Only directory generation is supported.") else: raise ValueError(f"Unknown command: {args.command}") diff --git a/flyde/cli.pyi b/flyde/cli.pyi deleted file mode 100644 index 537a5ce..0000000 --- a/flyde/cli.pyi +++ /dev/null @@ -1,11 +0,0 @@ -from _typeshed import Incomplete -from flyde.flow import Flow as Flow, add_folder_to_path as add_folder_to_path -from flyde.node import Component as Component - -log_level: Incomplete -logger: Incomplete - -def py_path_to_module(py_path: str) -> str: ... -def gen(path: str): - """Generate TypeScript files for a module.""" -def main() -> None: ... diff --git a/flyde/flow.py b/flyde/flow.py index 1f04796..d677069 100644 --- a/flyde/flow.py +++ b/flyde/flow.py @@ -1,26 +1,33 @@ import importlib +import importlib.util import logging import os import sys +from threading import Event from typing import Callable + import yaml # type: ignore -from threading import Event -from flyde.node import Graph +from flyde.node import Graph, InstanceArgs, InstanceType logger = logging.getLogger(__name__) class Flow: """Flow is a root-level runnable directed acyclic graph of nodes.""" + def __init__(self, imports: dict[str, list[str]]): self._imports = imports self._path = "" + self._base_path = "" self._node: Graph self._components: dict[str, Callable] = {} self._graphs: dict[str, dict] = {} def _preload_imports(self, base_path: str, imports: dict[str, list[str]]): + if not imports: + return + for module, classes in imports.items(): logger.debug(f"Importing {module}") # If module name ends with .flyde it's a Graph @@ -35,32 +42,126 @@ def _preload_imports(self, base_path: str, imports: dict[str, list[str]]): # Save the blueprint YAML for the graph to be instantiated later self._graphs[node_id] = yml["node"] continue - # Translate typescript file path to python module - module = ( - module.replace("/", ".").replace(".flyde.ts", "").replace("@", "") - ) + # Convert module path format + module = module.replace("/", ".").replace("@", "") logger.debug(f"Importing module {module}") mod = importlib.import_module(module) for class_name in classes: logger.debug(f"Importing {class_name} from {module}") self._components[class_name] = getattr(mod, class_name) - def factory(self, class_name: str, args: dict): + def _load_graph(self, name: str, path: str): + """Loads a graph YAML.""" + full_path = os.path.join(self._base_path, path) + yml = load_yaml_file(full_path) + if not isinstance(yml, dict): + raise ValueError(f"Invalid YAML file {path}") + # Save the blueprint YAML for the graph to be instantiated later + self._graphs[name] = yml["node"] + return + + def _load_component(self, name: str, path: str): + """Loads a component from a Python module.""" + # If component is already loaded, return + if name in self._components: + return + + # Handle custom://path/to/mod.py/ClassName format + if path.startswith("custom://"): + custom_path = path[9:] # Remove "custom://" prefix + if "/" in custom_path and custom_path.count("/") >= 1: + # Split into module path and class name + parts = custom_path.rsplit("/", 1) + if len(parts) == 2: + module_path, class_name = parts + + # Resolve the module path relative to the flow file's directory + if module_path.endswith(".py"): + # It's a file path, resolve it relative to the flow file directory + absolute_module_path = os.path.join( + self._base_path, module_path + ) + # Convert to module name for importing + spec = importlib.util.spec_from_file_location( + class_name, absolute_module_path + ) + if spec and spec.loader: + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + self._components[name] = getattr(mod, class_name) + return + else: + # It's already a module path, convert file path to module path + module_path = module_path.replace("/", ".").replace(".py", "") + + # Add the flow file's directory to sys.path temporarily for relative imports + original_path = sys.path[:] + if self._base_path not in sys.path: + sys.path.insert(0, self._base_path) + + try: + logger.debug( + f"Importing custom module {module_path}, class {class_name}" + ) + mod = importlib.import_module(module_path) + self._components[name] = getattr(mod, class_name) + return + finally: + # Restore original sys.path + sys.path[:] = original_path + + # Handle @flyde/nodes package format for stdlib components + if path == "@flyde/nodes": + logger.debug(f"Loading stdlib component {name}") + from flyde.nodes import Conditional, GetAttribute, Http, InlineValue + + stdlib_components = { + "InlineValue": InlineValue, + "Conditional": Conditional, + "GetAttribute": GetAttribute, + "Http": Http, + } + if name in stdlib_components: + self._components[name] = stdlib_components[name] + return + + raise ValueError( + f"Invalid component source path: {path}. Only custom:// and @flyde/nodes formats are supported." + ) + + def create_graph(self, name: str, args: InstanceArgs): + if name not in self._graphs: + if args.source is None: + raise ValueError(f"Graph {name} does not have a valid source") + self._load_graph(name, args.source.data) + + # Merge the blueprint YAML with the arguments + yml = self._graphs[name] | args.to_dict() + node = Graph.from_yaml(self.factory, yml) + return node + + def create_component(self, name: str, args: InstanceArgs): + if name not in self._components: + if args.source is None: + raise ValueError(f"Component {name} does not have a valid source") + self._load_component(name, args.source.data) + + if name not in self._components: + raise ValueError(f"Component {name} could not be loaded") + + # Create the component instance + component = self._components[name] + return component(**args.to_dict()) + + def factory(self, class_name: str, args: InstanceArgs): """Factory method to create a node from a class name and arguments. - It is used by the runtime to create nodes from the YAML definition or on the fly.""" - if class_name in self._graphs: - # Merge the blueprint YAML with the arguments - yml = self._graphs[class_name] | args - node = Graph.from_yaml(self.factory, yml) - return node - - # Look up the class in the imports - if class_name in self._components: - component = self._components[class_name] - return component(**args) + It is used by the runtime to create nodes from the YAML definition or on the fly. + """ + if args.type == InstanceType.VISUAL: + return self.create_graph(class_name, args) - raise ValueError(f"Unknown class name: {class_name}") + return self.create_component(class_name, args) def run(self): """Start the flow running. This is a non-blocking call as the flow runs in a separate thread.""" @@ -91,9 +192,10 @@ def from_yaml(cls, path: str, yml: dict): raise ValueError("No node in flow definition") ins = cls(imports) - ins._preload_imports(os.path.dirname(path), yml.get("imports", {})) + ins._path = path + ins._base_path = os.path.dirname(path) ins._node = Graph.from_yaml(ins.factory, yml["node"]) - ins._node.stopped = Event() + ins._node._stopped = Event() return ins @classmethod diff --git a/flyde/flow.pyi b/flyde/flow.pyi deleted file mode 100644 index 68d78c3..0000000 --- a/flyde/flow.pyi +++ /dev/null @@ -1,39 +0,0 @@ -from _typeshed import Incomplete -from flyde.node import Graph as Graph -from threading import Event - -logger: Incomplete - -class Flow: - """Flow is a root-level runnable directed acyclic graph of nodes.""" - _imports: Incomplete - _path: str - _node: Incomplete - _components: Incomplete - _graphs: Incomplete - def __init__(self, imports: dict[str, list[str]]) -> None: ... - def _preload_imports(self, base_path: str, imports: dict[str, list[str]]): ... - def factory(self, class_name: str, args: dict): - """Factory method to create a node from a class name and arguments. - - It is used by the runtime to create nodes from the YAML definition or on the fly.""" - def run(self) -> None: - """Start the flow running. This is a non-blocking call as the flow runs in a separate thread.""" - def run_sync(self) -> None: - """Run the flow synchronously. Shutdown handlers will be executed after the flow has finished.""" - @property - def node(self) -> Graph: - """The root node of the flow.""" - @property - def stopped(self) -> Event: - """Stopped event is set when the flow has finished working.""" - @classmethod - def from_yaml(cls, path: str, yml: dict): - """Load Flyde Flow definition from parsed YAML dict.""" - @classmethod - def from_file(cls, path: str): - """Load Flyde Flow definition from a *.flyde YAML file.""" - def to_dict(self) -> dict: ... - -def add_folder_to_path(path: str): ... -def load_yaml_file(yaml_file: str) -> dict: ... diff --git a/flyde/io.py b/flyde/io.py index ca5ef9e..1a53792 100644 --- a/flyde/io.py +++ b/flyde/io.py @@ -1,7 +1,8 @@ from copy import deepcopy +from dataclasses import dataclass from enum import Enum -from typing import Any, Optional from queue import Queue +from typing import Any, Optional EOF = Exception("__EOF__") """EOF is a signal to indicate the end of data.""" @@ -12,6 +13,24 @@ def is_EOF(value: Any) -> bool: return isinstance(value, Exception) and value.args[0] == "__EOF__" +class InputType(Enum): + """Input type contains all input types supported by Flyde.""" + + DYNAMIC = "dynamic" + NUMBER = "number" + BOOLEAN = "boolean" + JSON = "json" + STRING = "string" + + +@dataclass +class InputConfig: + """Configuration of an input in a Flyde flow.""" + + type: InputType + value: Optional[Any] = None + + class InputMode(Enum): """InputMode is the mode of an input. @@ -115,14 +134,14 @@ def value(self, value: Any): def get(self) -> Any: """Get the value of the input from either the queue or static value.""" if not self.is_connected and ( - self.required == Requiredness.OPTIONAL or - self.required == Requiredness.REQUIRED_IF_CONNECTED): + self.required == Requiredness.OPTIONAL or self.required == Requiredness.REQUIRED_IF_CONNECTED + ): return self._value if self._input_mode == InputMode.QUEUE: - return self._queue.get() + return self.queue.get() elif self._input_mode == InputMode.STICKY: - if not self._queue.empty() or self._value is None: - value = self._queue.get() + if not self.queue.empty() or self._value is None: + value = self.queue.get() if not is_EOF(value): # Ignore EOFs on sticky inputs, only queue inputs matter for termination self._value = value @@ -153,6 +172,26 @@ def ref_count(self) -> int: """Get the reference count of the input.""" return self._ref_count + def apply_config(self, config: InputConfig): + """Apply config from the flyde flow to the input.""" + self._value = config.value + # If input mode is STICKY already, stick to it + # as it may be important for the node to function correctly + if self._input_mode != InputMode.STICKY: + if config.type == InputType.DYNAMIC: + self._input_mode = InputMode.QUEUE + else: + self._input_mode = InputMode.STICKY + + # Apply Python type hint based on supported config type + if config.type != InputType.DYNAMIC and self.type is None: + self.type = { + InputType.NUMBER: int, + InputType.BOOLEAN: bool, + InputType.JSON: dict, + InputType.STRING: str, + }[config.type] + class Output: """Output is an interface for setting output data for a component.""" @@ -197,9 +236,7 @@ def connected(self) -> bool: def send(self, value: Any): """Put a value in the output queue.""" if self.type is not None and not is_EOF(value) and not isinstance(value, self.type): # type: ignore - raise ValueError( - f'Output "{self.id}": value {value} is not of type {self.type}' - ) + raise ValueError(f'Output "{self.id}": value {value} is not of type {self.type}') if len(self._queues) == 0: raise ValueError(f'Output "{self.id}": has no connected queues') @@ -294,11 +331,11 @@ def __init__( def inc_ref_count(self): # Need to increase ref count of the RedriveQueue - self._queue.inc_ref_count() # type: ignore + self._queue.inc_ref_count() # type: ignore return super().inc_ref_count() def dec_ref_count(self): - self._queue.dec_ref_count() # type: ignore + self._queue.dec_ref_count() # type: ignore return super().dec_ref_count() diff --git a/flyde/io.pyi b/flyde/io.pyi deleted file mode 100644 index 9a35e0d..0000000 --- a/flyde/io.pyi +++ /dev/null @@ -1,161 +0,0 @@ -from _typeshed import Incomplete -from enum import Enum -from queue import Queue -from typing import Any - -EOF: Incomplete - -def is_EOF(value: Any) -> bool: - """Checks if a value is an EOF signal.""" - -class InputMode(Enum): - """InputMode is the mode of an input. - - QUEUE: The input is connected to a queue. On each node invocation, a new value is taken from the queue. - If the queue is empty, the node invocation is blocked. - STICKY: The input has a sticky value. It has a queue attached to it, but the last received value is returned in - absence of new values in the queue. Thus sticky inputs are non-blocking. - STATIC: The input has a static value that does not change.""" - QUEUE = 'queue' - STICKY = 'sticky' - STATIC = 'static' - -class Requiredness(Enum): - """Requiredness of an input. - - REQUIRED: The input is required to be connected. - OPTIONAL: The input is optional. - REQUIRED_IF_CONNECTED: The input is required if it is connected to a queue.""" - REQUIRED = 'required' - OPTIONAL = 'optional' - REQUIRED_IF_CONNECTED = 'required-if-connected' - -class OutputMode(Enum): - """OutputMode defines the behavior of an output if it is connected to multiple input queues. - - REF: Copy-by-reference. Each connected input will receive the same object. - VALUE: Copy-by-value. Each connected input will receive a deep copy of the object. - CIRCLE: Circular. Each connected input will receive the object in a round-robin fashion. - """ - REF = 'ref' - VALUE = 'value' - CIRCLE = 'circle' - -class Input: - """Input is an interface for getting input/output data for a node.""" - id: Incomplete - description: Incomplete - type: Incomplete - _input_mode: Incomplete - _value: Incomplete - required: Incomplete - _ref_count: int - def __init__(self, /, id: str = '', description: str = '', mode: InputMode = ..., type: type | None = None, value: Any = None, required: Requiredness = ...) -> None: - """Create a new input object. - - Args: - id (str): The ID of the input - description (str): The description of the input - mode (InputMode): The mode of the input - typ (type): The type of the input - value (Any): The value of the input for InputMode = InputMode.STATIC or InputMode = InputMode.STICKY - required (Required): The requiredness of the input - """ - _queue: Incomplete - @property - def queue(self) -> Queue: - """Get the queue of the input.""" - @property - def is_connected(self) -> bool: - """Check if the input is connected to a queue.""" - @property - def value(self) -> Any: - """Get the static value associated with the input.""" - @value.setter - def value(self, value: Any): - """Set the static value of the input.""" - def get(self) -> Any: - """Get the value of the input from either the queue or static value.""" - def empty(self) -> bool: - """Check if the input queue is empty.""" - def count(self) -> int: - """Get the number of elements in the input queue.""" - def inc_ref_count(self) -> None: - """Increment the reference count of the input.""" - def dec_ref_count(self) -> None: - """Decrement the reference count of the input.""" - @property - def ref_count(self) -> int: - """Get the reference count of the input.""" - -class Output: - """Output is an interface for setting output data for a component.""" - id: Incomplete - description: Incomplete - _output_mode: Incomplete - type: Incomplete - delayed: Incomplete - _queues: Incomplete - _circle_index: int - def __init__(self, /, id: str = '', description: str = '', mode: OutputMode = ..., type: type | None = None, delayed: bool = False) -> None: - """Create a new output object. - - Args: - id (str): The ID of the output - description (str): The description of the output - type (type): The type of the output - delayed (bool): If the output is delayed [not implemented yet] - """ - def connect(self, queue: Queue): - """Connect a queue to the output. - - This method can be called multiple times to connect multiple queues to the same output. - """ - @property - def connected(self) -> bool: - """Check if the output is connected to a queue.""" - def send(self, value: Any): - """Put a value in the output queue.""" - -class RedirectQueue: - """RedriveQueue is a fake write-only queue that is used by GraphPort - to redrive input values to the output queues.""" - _output: Incomplete - _ref_count: int - def __init__(self, output: Output) -> None: ... - @property - def ref_count(self) -> int: ... - def inc_ref_count(self) -> None: ... - def dec_ref_count(self) -> None: ... - def put(self, item: Any, block: bool = True, timeout: Incomplete | None = None): ... - -class GraphPort(Input, Output): - """GraphPort is an interface between inside and outside of the graph used for input/output. - - It combines Input and Output, because Graph Input acts as an Input for outside world, - but outputs values inside the graph. Similarly, Graph Output acts as an Output for outside world, - but receives values from inside the graph.""" - _queue: Incomplete - def __init__(self, id: str = '', description: str = '', type: type | None = None, value: Any = None, required: Requiredness = ..., output_mode: OutputMode = ..., delayed: bool = False) -> None: ... - def inc_ref_count(self): ... - def dec_ref_count(self): ... - -class ConnectionNode: - """ConnectionNode is a combination of a node and an input/output pin. - - It is used as a source or destination of a connection.""" - ins_id: Incomplete - pin_id: Incomplete - def __init__(self, ins_id: str, pin_id: str) -> None: ... - -class Connection: - """Connection is a connection between two nodes in a graph.""" - from_node: Incomplete - to_node: Incomplete - delayed: Incomplete - hidden: Incomplete - def __init__(self, from_node: ConnectionNode, to_node: ConnectionNode, delayed: bool = False, hidden: bool = False) -> None: ... - @classmethod - def from_yaml(cls, yml: dict): - """Create a connection from a parsed YAML dictionary.""" - def to_dict(self) -> dict: ... diff --git a/flyde/node.py b/flyde/node.py index d01545e..75ecc48 100644 --- a/flyde/node.py +++ b/flyde/node.py @@ -1,19 +1,72 @@ import logging from abc import ABC, abstractmethod from copy import deepcopy +from dataclasses import dataclass +from enum import Enum from threading import Event, Lock, Thread -from typing import Any, Callable +from typing import Any, Callable, Optional from uuid import uuid4 -from flyde.io import GraphPort, InputMode, Input, Output, EOF, Requiredness, is_EOF, Connection +from flyde.io import EOF, Connection, GraphPort, Input, InputConfig, InputMode, InputType, Output, Requiredness, is_EOF logger = logging.getLogger(__name__) -SUPPORTED_MACROS = ["InlineValue", "Conditional", "GetAttribute"] + +class InstanceType(Enum): + """InstanceType is the type of an instance. + + VISUAL: The instance is a visual node. + CODE: The instance is a code node. + """ + + VISUAL = "visual" + CODE = "code" + + +class InstanceSourceType(Enum): + """InstanceSourceType is the source type of an instance. + + FILE: The instance is created from a file. + PACKAGE: The instance is created from a built in package. + CUSTOM: The instance is created from a custom module with path format.""" + + FILE = "file" + PACKAGE = "package" + CUSTOM = "custom" + + +@dataclass +class InstanceSource: + """Source configuration of an instance.""" + + type: InstanceSourceType + data: str + + +@dataclass +class InstanceArgs: + """Arguments to pass to the instance factory.""" + + id: str + display_name: str + stopped: Optional[Event] + config: dict[str, Any] + type: InstanceType = InstanceType.CODE + source: Optional[InstanceSource] = None + + def to_dict(self) -> dict: + """Convert the instance arguments to a dictionary.""" + return { + "id": self.id, + "display_name": self.display_name, + "stopped": self.stopped, + "config": self.config, + } + # InstanceFactory is a function that creates a new instance of a node. # It can create instances dynamically based on the node ID. -InstanceFactory = Callable[[str, dict], Any] +InstanceFactory = Callable[[str, InstanceArgs], Any] class Node(ABC): @@ -22,7 +75,7 @@ class Node(ABC): Attributes: id (str): A unique identifier for the node. node_type (str): The node type identifier. - input_config (dict): A dictionary of input pin configurations. + config (dict): A dictionary of input pin configurations. display_name (str): A human-readable name for the node. inputs (dict[str, Input]): Node input map. outputs (dict[str, Output]): Node output map. @@ -36,17 +89,18 @@ def __init__( /, id: str, node_type: str = "", - input_config: dict[str, InputMode] = {}, display_name: str = "", inputs: dict[str, Input] = {}, outputs: dict[str, Output] = {}, stopped: Event = Event(), + config: dict[str, InputConfig] = {}, ): node_type = node_type if node_type else self.__class__.__name__ self._node_type = node_type self._id = id if id else create_instance_id(node_type) - self._input_config = input_config self._display_name = display_name if display_name else node_type + self._config_raw = config or {} + self._config = self.parse_config(self._config_raw) if len(inputs) > 0: self.inputs = inputs @@ -58,6 +112,8 @@ def __init__( for k, v in self.inputs.items(): v.id = f"{self._id}.{k}" + if k in self._config: + v.apply_config(self._config[k]) if len(outputs) > 0: self.outputs = outputs @@ -72,6 +128,20 @@ def __init__( self._stopped = stopped + def parse_config(self, config: dict[str, Any]) -> dict[str, Any]: + """Parse the raw config into a typed config dictionary.""" + result = {} + for key, value in config.items(): + if isinstance(value, dict) and "type" in value and value["type"] in [item.value for item in InputType]: + config_value = value.get("value", None) + result[key] = InputConfig( + type=InputType(value["type"]), + value=config_value, + ) + else: + result[key] = value + return result + @abstractmethod def run(self): """Run the node. This method should be overridden by subclasses.""" @@ -121,22 +191,29 @@ def from_yaml(cls, create: InstanceFactory, yml: dict): node_class_name = yml.get("nodeId", "VisualNode") if node_class_name == "VisualNode": return Graph.from_yaml(create, yml) - args = { - "id": yml["id"], - "input_config": yml.get("inputConfig", {}), - "display_name": yml.get("displayName", ""), - "stopped": yml.get("stopped", None), # It's a hacky way to pass the stopped event to the constructor - } - # If macro parameters are present, pass them to the constructor - if "macroData" in yml: - args["macro_data"] = yml["macroData"] + + config = yml.get("config", {}) + + source = InstanceSource( + type=InstanceSourceType(yml.get("source", {}).get("type", "file").lower()), + data=yml.get("source", {}).get("data", ""), + ) + + args = InstanceArgs( + id=yml["id"], + display_name=yml.get("displayName", ""), + stopped=yml.get("stopped", None), # It's a hacky way to pass the stopped event to the constructor + config=config, + type=InstanceType(yml.get("type", "code").lower()), + source=source, + ) return create(node_class_name, args) def to_dict(self) -> dict: return { "id": self._id, "nodeId": self._node_type, - "inputConfig": self._input_config, + "config": self._config, "displayName": self._display_name, } @@ -156,18 +233,28 @@ def __init__(self, **kwargs): def run(self): if not hasattr(self, "process"): - raise NotImplementedError( - "Component does not have neither run() nor process() method. No code to run." - ) + raise NotImplementedError("Component does not have neither run() nor process() method. No code to run.") def worker(): logger.debug(f"Running {self._id} worker") + + # Check if all inputs are sticky or static (not queue) + # If so, we only run the loop once + all_sticky_or_static = True + for inp in self.inputs.values(): + if inp._input_mode == InputMode.QUEUE: + all_sticky_or_static = False + break + + run_once = len(self.inputs) > 0 and all_sticky_or_static + while not self._stop.is_set(): logger.debug(f"Waiting for inputs on {self._id}") inputs = {} queue_count = 0 queue_closed_count = 0 skip_iteration = False + for key, inp in self.inputs.items(): is_queue = inp._input_mode == InputMode.QUEUE value = inp.get() @@ -198,9 +285,7 @@ def worker(): logger.debug(f"Processing {self._id} with inputs: {inputs}") res = self.process(**inputs) # type: ignore - if isinstance(res, dict) or ( - isinstance(res, tuple) and hasattr(res, "_fields") - ): + if isinstance(res, dict) or (isinstance(res, tuple) and hasattr(res, "_fields")): # Send values to the outputs named as keys for k, v in res.items(): # type: ignore if k not in self.outputs: @@ -217,6 +302,11 @@ def worker(): if self.outputs[k].connected: self.outputs[k].send(v) + # If all inputs are sticky/static, exit after the first iteration + if run_once: + logger.debug(f"All inputs are sticky or static for {self._id}, stopping after first execution") + break + self.finish() logger.debug(f"Starting {self._id} thread") @@ -228,55 +318,6 @@ def stop(self): logger.debug(f"Stopping {self._id}") self._stop.set() - @classmethod - def to_ts(cls, name: str = "") -> str: - """Convert the node to a TypeScript definition.""" - - name = cls.__name__ if name == "" else name # type: ignore - - inputs_str = "" - if hasattr(cls, "inputs") and len(cls.inputs) > 0: - inputs_str = ( - "\n" - + ",\n".join( - [ - f' {k}: {{ description: "{v.description}" }}' - for k, v in cls.inputs.items() - ] - ) - + "\n" - ) - outputs_str = "" - if hasattr(cls, "outputs") and len(cls.outputs) > 0: - outputs_str = ( - "\n" - + ",\n".join( - [ - f' {k}: {{ description: "{v.description}" }}' - for k, v in cls.outputs.items() - ] - ) - + "\n" - ) - - safe_doc = "" - if hasattr(cls, "__doc__") and cls.__doc__: - safe_doc = ( - cls.__doc__.replace("\n", "\\n") - .replace("\r", "\\r") - .replace('"', '\\"') - ) - - return ( - f"export const {name}: CodeNode = {{\n" - f' id: "{name}",\n' - f' description: "{safe_doc}",\n' - f" inputs: {{{inputs_str} }},\n" - f" outputs: {{{outputs_str} }},\n" - f" run: () => {{ return; }},\n" - f"}};\n\n" - ) - class Graph(Node): """A visual graph node that contains other nodes.""" @@ -286,7 +327,7 @@ def __init__( /, id: str = "", node_type: str = "", - input_config: dict[str, InputMode] = {}, + config: dict[str, InputConfig] = {}, display_name: str = "", instances: dict[str, Node] = {}, instances_stopped: dict[str, Event] = {}, @@ -298,7 +339,7 @@ def __init__( super().__init__( id=id, node_type=node_type, - input_config=input_config, + config=config, display_name=display_name, stopped=stopped, ) @@ -321,13 +362,13 @@ def __init__( if from_pin not in self.inputs: raise ValueError(f"Input {from_pin} not found in graph {self._id}") else: - self._check_pin('out', from_id, from_pin) + self._check_pin("out", from_id, from_pin) if to_id == "__this": if to_pin not in self.outputs: raise ValueError(f"Output {to_pin} not found in graph {self._id}") else: - self._check_pin('in', to_id, to_pin) + self._check_pin("in", to_id, to_pin) if from_id != "__this" and to_id != "__this": # Simple case: connect two instances inside the graph @@ -362,9 +403,7 @@ def _check_pin(self, pin_type: str, instance_id: str, pin_id: str): def run(self): """Run the graph.""" for instance in self._instances.values(): - logger.debug( - f"Running instance {instance._id} of type {instance._node_type}" - ) + logger.debug(f"Running instance {instance._id} of type {instance._node_type}") instance.run() def worker(): @@ -420,7 +459,7 @@ def from_yaml(cls, create: InstanceFactory, yml: dict): # Load metadata node_type = yml.get("nodeId", __name__) id = yml["id"] if "id" in yml else create_instance_id(node_type) - input_config = yml.get("inputConfig", {}) + config = {k: InputConfig(**v) for k, v in yml.get("config", {}).items()} display_name = yml.get("displayName", node_type) # Load instances and macros @@ -428,11 +467,6 @@ def from_yaml(cls, create: InstanceFactory, yml: dict): instances_stopped = {} for ins in yml.get("instances", []): ins_id = ins["id"] - if "macroId" in ins: - # Only InlineValue macros are supported for now - if ins["macroId"] not in SUPPORTED_MACROS: - raise ValueError(f'Unsupported macro: {ins["macroId"]}') - ins["nodeId"] = ins["macroId"] stopped = Event() ins["stopped"] = stopped logger.debug(f"Creating instance {ins_id}") @@ -441,9 +475,7 @@ def from_yaml(cls, create: InstanceFactory, yml: dict): logger.debug(f"Loaded instance {ins_id}") # Load connections and graph inputs/outputs - connections = [ - Connection.from_yaml(conn) for conn in yml.get("connections", []) - ] + connections = [Connection.from_yaml(conn) for conn in yml.get("connections", [])] inputs = {} for k, v in yml.get("inputs", {}).items(): if "mode" in v: @@ -462,7 +494,7 @@ def from_yaml(cls, create: InstanceFactory, yml: dict): return cls( id=id, node_type=node_type, - input_config=input_config, + config=config, display_name=display_name, instances=instances, instances_stopped=instances_stopped, @@ -477,7 +509,7 @@ def to_dict(self) -> dict: return { "id": self._id, "nodeId": self._node_type, - "inputConfig": self._input_config, + "config": self._config, "displayName": self._display_name, "inputs": self.inputs, "outputs": self.outputs, diff --git a/flyde/node.pyi b/flyde/node.pyi deleted file mode 100644 index 6fae1ec..0000000 --- a/flyde/node.pyi +++ /dev/null @@ -1,98 +0,0 @@ -import abc -from _typeshed import Incomplete -from abc import ABC, abstractmethod -from flyde.io import Connection as Connection, EOF as EOF, GraphPort as GraphPort, Input as Input, InputMode as InputMode, Output as Output, Requiredness as Requiredness, is_EOF as is_EOF -from threading import Event -from typing import Any, Callable - -logger: Incomplete -SUPPORTED_MACROS: Incomplete -InstanceFactory = Callable[[str, dict], Any] - -class Node(ABC, metaclass=abc.ABCMeta): - """Node is the main building block of an application. - - Attributes: - id (str): A unique identifier for the node. - node_type (str): The node type identifier. - input_config (dict): A dictionary of input pin configurations. - display_name (str): A human-readable name for the node. - inputs (dict[str, Input]): Node input map. - outputs (dict[str, Output]): Node output map. - """ - inputs: dict[str, Input] - outputs: dict[str, Output] - _node_type: Incomplete - _id: Incomplete - _input_config: Incomplete - _display_name: Incomplete - _stopped: Incomplete - def __init__(self, /, id: str, node_type: str = '', input_config: dict[str, InputMode] = {}, display_name: str = '', inputs: dict[str, Input] = {}, outputs: dict[str, Output] = {}, stopped: Event = ...) -> None: ... - @abstractmethod - def run(self): - """Run the node. This method should be overridden by subclasses.""" - @abstractmethod - def stop(self): - """Stop the node. This method should be overridden by subclasses.""" - def finish(self) -> None: - """Finish the component execution gracefully by closing all its outputs and notifying others.""" - @property - def stopped(self) -> Event: ... - def shutdown(self) -> None: - """Shutdown the component. This method is optional and can be overridden by subclasses.""" - def send(self, output_id: str, value: Any): - """Send a value to an output.""" - def receive(self, input_id: str) -> Any: - """Receive a value from an input.""" - @classmethod - def from_yaml(cls, create: InstanceFactory, yml: dict): - """Create a node from a parsed YAML dictionary.""" - def to_dict(self) -> dict: ... - -class Component(Node): - """A node that runs a function when executed.""" - _stop: Incomplete - _mutex: Incomplete - def __init__(self, **kwargs) -> None: ... - def run(self) -> None: ... - def stop(self) -> None: - """Stop the component execution.""" - @classmethod - def to_ts(cls, name: str = '') -> str: - """Convert the node to a TypeScript definition.""" - -class Graph(Node): - """A visual graph node that contains other nodes.""" - inputs: Incomplete - outputs: Incomplete - _connections: Incomplete - _instances: Incomplete - _instances_stopped: Incomplete - def __init__(self, /, id: str = '', node_type: str = '', input_config: dict[str, InputMode] = {}, display_name: str = '', instances: dict[str, Node] = {}, instances_stopped: dict[str, Event] = {}, connections: list[Connection] = [], inputs: dict[str, GraphPort] = {}, outputs: dict[str, GraphPort] = {}, stopped: Event = ...) -> None: ... - def _check_pin(self, pin_type: str, instance_id: str, pin_id: str): - """Check if the instance and pin exist.""" - def run(self) -> None: - """Run the graph.""" - def shutdown(self) -> None: - """Call shutdown handlers on all instances. - - This method is called from the main thread to allow cleanup and things like UI.""" - def stop(self) -> None: - """Stop all instances gracefully.""" - def terminate(self) -> None: - """Terminate all instances immediately.""" - @property - def stopped(self) -> Event: - """Return the stopped event which is set when the node has stopped.""" - _stopped: Incomplete - @stopped.setter - def stopped(self, value: Event): - """Set the stopped event.""" - @classmethod - def from_yaml(cls, create: InstanceFactory, yml: dict): - """Create a Graph node from a parsed YAML dictionary.""" - def to_dict(self) -> dict: - """Return a dictionary representation of the node.""" - -def create_instance_id(node_type: str) -> str: - """Create a unique instance ID.""" diff --git a/flyde/nodes.py b/flyde/nodes.py new file mode 100644 index 0000000..a64032d --- /dev/null +++ b/flyde/nodes.py @@ -0,0 +1,326 @@ +import inspect +import json +import re +from dataclasses import dataclass +from enum import Enum +from typing import Any, Optional, Union +from urllib import error, parse, request + +from flyde.io import Input, InputConfig, InputMode, InputType, Output, Requiredness +from flyde.node import Component + + +def list_nodes(): + """Dynamically discover all Component classes defined in this module. + + Returns: + list: List of class names that inherit from Component + """ + current_module = inspect.getmodule(inspect.currentframe()) + component_classes = [] + + for name, obj in inspect.getmembers(current_module): + if ( + inspect.isclass(obj) + and issubclass(obj, Component) + and obj != Component + and obj.__module__ == current_module.__name__ + ): + component_classes.append(name) + + return sorted(component_classes) + + +class InlineValue(Component): + """InlineValue sends a constant value to output.""" + + icon = "pencil" + outputs = {"value": Output(description="The constant value")} + + def __init__(self, **kwargs): + super().__init__(**kwargs) + if "value" in self._config: + value = self._config["value"] + self.value = value.value + else: + raise ValueError("Missing value in InlineValue configuration.") + + def process(self): + self.send("value", self.value) + # Inline value only runs once + self.stop() + + +class _ConditionType(Enum): + """Condition type enumeration.""" + + Equal = "EQUAL" + NotEqual = "NOT_EQUAL" + Contains = "CONTAINS" + NotContains = "NOT_CONTAINS" + RegexMatches = "REGEX_MATCHES" + Exists = "EXISTS" + NotExists = "NOT_EXISTS" + + +@dataclass +class _ConditionConfig: + """Configuration etry for the condition type.""" + + type: _ConditionType + + +class _ConditionalConfig: + """Conditional configuration.""" + + def __init__(self, config: dict[str, Union[InputConfig, _ConditionConfig]]): + if "condition" not in config: + raise ValueError("Missing 'condition' in Conditional configuration.") + if not isinstance(config["condition"], _ConditionConfig): + raise ValueError("Invalid 'condition' in Conditional configuration.") + condition = config["condition"] + self.condition_type = _ConditionType(condition.type) + + if "leftOperand" in config and isinstance(config["leftOperand"], InputConfig): + self.left_operand: InputConfig = config["leftOperand"] + + if "rightOperand" in config and isinstance(config["rightOperand"], InputConfig): + self.right_operand = config["rightOperand"] + + +class Conditional(Component): + """Conditional component evaluates a condition against the input and sends the result to output.""" + + icon = "circle-question" + inputs = { + "leftOperand": Input(description="Left operand of the condition"), + "rightOperand": Input(description="Right operand of the condition"), + } + outputs = { + "true": Output(description="Output when the condition is true"), + "false": Output(description="Output when the condition is false"), + } + + def parse_config(self, config: dict[str, Any]) -> dict[str, Any]: + """Parse the raw config, handling the 'condition' special case.""" + result = super().parse_config(config) # type: ignore + + # Handle the condition special case + if "condition" in result and isinstance(result["condition"], dict) and "type" in result["condition"]: + result["condition"] = _ConditionConfig(**result["condition"]) + + return result + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._config = _ConditionalConfig(self._config) + if hasattr(self._config, "left_operand") and self._config.left_operand.type != InputType.DYNAMIC: + self.inputs["leftOperand"]._input_mode = InputMode.STATIC + self.inputs["leftOperand"].value = self._config.left_operand.value + if hasattr(self._config, "right_operand") and self._config.right_operand.type != InputType.DYNAMIC: + self.inputs["rightOperand"]._input_mode = InputMode.STATIC + self.inputs["rightOperand"].value = self._config.right_operand.value + + def _evaluate(self, left_operand: Any, right_operand: Any) -> bool: + condition_type = self._config.condition_type + if condition_type == _ConditionType.Equal: + return left_operand == right_operand + elif condition_type == _ConditionType.NotEqual: + return left_operand != right_operand + elif condition_type == _ConditionType.Contains: + return right_operand in left_operand + elif condition_type == _ConditionType.NotContains: + return right_operand not in left_operand + elif condition_type == _ConditionType.RegexMatches: + m = re.match(right_operand, left_operand) + return m is not None + elif condition_type == _ConditionType.Exists: + return left_operand is not None and left_operand != "" and left_operand != [] + elif condition_type == _ConditionType.NotExists: + return left_operand is None or left_operand == "" or left_operand == [] + else: + raise ValueError(f"Unsupported condition type: {condition_type}") + + def process(self, leftOperand: Any, rightOperand: Any): + result = self._evaluate(leftOperand, rightOperand) + + if result: + self.send("true", leftOperand) + else: + self.send("false", leftOperand) + + +class GetAttribute(Component): + """Get an attribute from an object or dictionary.""" + + icon = "fa-magnifying-glass" + inputs = { + "object": Input(description="The object or dictionary"), + "key": Input(description="The attribute name", type=str), + } + outputs = {"value": Output(description="The attribute value")} + + def __init__(self, **kwargs): + super().__init__(**kwargs) + if "key" not in self._config: + raise ValueError("Missing 'key' in GetAttribute configuration.") + key = self._config["key"] + if not isinstance(key, InputConfig): + raise ValueError("Invalid 'key' in GetAttribute configuration.") + if key.type == InputType.DYNAMIC: + self.inputs["key"]._input_mode = InputMode.STICKY # type: ignore + if key.value is not None: + self.inputs["key"].value = key.value + else: + self.inputs["key"]._input_mode = InputMode.STATIC # type: ignore + self.inputs["key"].value = key.value + + def process(self, object: Any, key: str): + keys = key.split(".") + value = object + for k in keys: + if isinstance(value, dict): + value = value.get(k, None) + elif hasattr(value, k): + value = getattr(value, k) + else: + value = None + break + self.send("value", value) + + +class Http(Component): + """Http component makes HTTP requests with urllib.""" + + icon = "globe" + inputs = { + "url": Input(description="URL to request", required=Requiredness.REQUIRED), + "method": Input(description="HTTP method", type=str, required=Requiredness.REQUIRED), + "headers": Input(description="HTTP headers", type=dict, required=Requiredness.OPTIONAL), + "params": Input(description="URL parameters", type=dict, required=Requiredness.OPTIONAL), + "data": Input(description="Request body", type=dict, required=Requiredness.OPTIONAL), + } + outputs = { + "data": Output(description="Response data"), + } + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + if "method" in self._config and isinstance(self._config["method"], InputConfig): + if self._config["method"].type == InputType.DYNAMIC: + self.inputs["method"]._input_mode = InputMode.STICKY + else: + self.inputs["method"]._input_mode = InputMode.STATIC + self.inputs["method"].value = self._config["method"].value + else: + self.inputs["method"]._input_mode = InputMode.STATIC + self.inputs["method"].value = "GET" + + if "url" in self._config and isinstance(self._config["url"], InputConfig): + if self._config["url"].type == InputType.DYNAMIC: + self.inputs["url"]._input_mode = InputMode.QUEUE + else: + self.inputs["url"]._input_mode = InputMode.STATIC + self.inputs["url"].value = self._config["url"].value + + if "headers" in self._config and isinstance(self._config["headers"], InputConfig): + if self._config["headers"].type == InputType.DYNAMIC: + self.inputs["headers"]._input_mode = InputMode.STICKY + else: + self.inputs["headers"]._input_mode = InputMode.STATIC + self.inputs["headers"].value = self._config["headers"].value + else: + self.inputs["headers"]._input_mode = InputMode.STATIC + self.inputs["headers"].value = {} + + if "params" in self._config and isinstance(self._config["params"], InputConfig): + if self._config["params"].type == InputType.DYNAMIC: + self.inputs["params"]._input_mode = InputMode.STICKY + else: + self.inputs["params"]._input_mode = InputMode.STATIC + self.inputs["params"].value = self._config["params"].value + else: + self.inputs["params"]._input_mode = InputMode.STATIC + self.inputs["params"].value = {} + + if "data" in self._config and isinstance(self._config["data"], InputConfig): + if self._config["data"].type == InputType.DYNAMIC: + self.inputs["data"]._input_mode = InputMode.STICKY + else: + self.inputs["data"]._input_mode = InputMode.STATIC + self.inputs["data"].value = self._config["data"].value + else: + self.inputs["data"]._input_mode = InputMode.STATIC + self.inputs["data"].value = {} + + def process( + self, + url: str, + method: str, + headers: Optional[dict] = None, + params: Optional[dict] = None, + data: Optional[dict] = None, + ): + try: + if params: + url_parts = list(parse.urlparse(url)) + query = dict(parse.parse_qsl(url_parts[4])) + query.update(params) + url_parts[4] = parse.urlencode(query) + url = parse.urlunparse(url_parts) + + req = request.Request(url) + req.method = method + + if headers: + for key, value in headers.items(): + req.add_header(key, value) + + data_bytes = None + if data and method != "GET": + data_bytes = json.dumps(data).encode("utf-8") + req.add_header("Content-Type", "application/json") + req.add_header("Content-Length", str(len(data_bytes))) + + with request.urlopen(req, data=data_bytes) as response: + content_type = response.headers.get("Content-Type", "") + response_data = response.read() + + # Handle text-based responses + if ( + "text/" in content_type + or "json" in content_type + or "xml" in content_type + or "application/javascript" in content_type + ): + # Extract charset from content-type header if present + charset = "utf-8" # Default charset + if "charset=" in content_type: + charset_part = content_type.split("charset=")[1] + if ";" in charset_part: + charset = charset_part.split(";")[0].strip() + else: + charset = charset_part.strip() + + try: + response_data = response_data.decode(charset) + except (UnicodeDecodeError, LookupError): + # Fallback to utf-8 if specified charset fails + response_data = response_data.decode("utf-8", errors="replace") + + # Try to parse JSON if the content type indicates JSON + if "json" in content_type: + try: + response_data = json.loads(response_data) + except json.JSONDecodeError: + pass + # Binary data remains as bytes + + self.send("data", response_data) + except error.HTTPError as e: + raise e + except error.URLError as e: + raise e + except Exception as e: + raise e diff --git a/flyde/__init__.pyi b/flyde/py.typed similarity index 100% rename from flyde/__init__.pyi rename to flyde/py.typed diff --git a/flyde/stdlib.py b/flyde/stdlib.py deleted file mode 100644 index c3d5c9c..0000000 --- a/flyde/stdlib.py +++ /dev/null @@ -1,163 +0,0 @@ -import re -from enum import Enum -from typing import Any - -from flyde.node import Component -from flyde.io import Input, Output, InputMode - - -class InlineValue(Component): - """InlineValue sends a constant value to output.""" - - outputs = {"value": Output(description="The constant value")} - - def __init__(self, macro_data: dict, **kwargs): - super().__init__(**kwargs) - if "value" in macro_data: - value = macro_data["value"] - self.value = self._get_inline_value(value) if self._is_inline_dict(value) else value - else: - raise ValueError("Missing value in InlineValue configuration.") - - def process(self): - self.send("value", self.value) - # Inline value only runs once - self.stop() - - def _is_inline_dict(self, value: Any) -> bool: - """Check if a value is an inline Flyde value dict, which has `type` and `value` keys.""" - supported_inline_types = ["dynamic", "string", "number", "boolean", "json", "select", "longtext"] - return isinstance(value, dict) and "type" in value and value["type"] in supported_inline_types - - def _get_inline_value(self, value: Any) -> Any: - """Get the value from an inline Flyde value output.""" - return value["value"] - - -class _ConditionType(Enum): - """Condition type enumeration.""" - - Equal = "EQUAL" - NotEqual = "NOT_EQUAL" - Contains = "CONTAINS" - NotContains = "NOT_CONTAINS" - RegexMatches = "REGEX_MATCHES" - Exists = "EXISTS" - NotExists = "NOT_EXISTS" - - -class _ConditionalConfig: - """Conditional configuration.""" - - def __init__(self, yml: dict): - self.property_path = yml.get("propertyPath", "") - - condition = yml.get("condition", {}) - condition_type = condition.get("type", "EQUAL") - try: - self.condition_type = _ConditionType(condition_type) - except ValueError: - raise ValueError(f"Unsupported condition type: {condition_type}") - self.condition_data = condition.get("data", "") - - left_operand = yml.get("leftOperand", {}) - self.left_operand = { - "type": left_operand.get("type", "dynamic"), - "value": left_operand.get("value", ""), - } - right_operand = yml.get("rightOperand", {}) - self.right_operand = { - "type": right_operand.get("type", "dynamic"), - "value": right_operand.get("value", ""), - } - - -class Conditional(Component): - """Conditional component evaluates a condition against the input and sends the result to output.""" - - inputs = { - "leftOperand": Input(description="Left operand of the condition"), - "rightOperand": Input(description="Right operand of the condition"), - } - outputs = { - "true": Output(description="Output when the condition is true"), - "false": Output(description="Output when the condition is false"), - } - - def __init__(self, macro_data: dict, **kwargs): - super().__init__(**kwargs) - self._config = _ConditionalConfig(macro_data) - if self._config.left_operand["type"] != "dynamic": - self.inputs["leftOperand"]._input_mode = InputMode.STATIC - self.inputs["leftOperand"].value = self._config.left_operand["value"] - if self._config.right_operand["type"] != "dynamic": - self.inputs["rightOperand"]._input_mode = InputMode.STATIC - self.inputs["rightOperand"].value = self._config.right_operand["value"] - - def _evaluate(self, left_operand: Any, right_operand: Any) -> bool: - condition_type = self._config.condition_type - if condition_type == _ConditionType.Equal: - return left_operand == right_operand - elif condition_type == _ConditionType.NotEqual: - return left_operand != right_operand - elif condition_type == _ConditionType.Contains: - return right_operand in left_operand - elif condition_type == _ConditionType.NotContains: - return right_operand not in left_operand - elif condition_type == _ConditionType.RegexMatches: - m = re.match(right_operand, left_operand) - return m is not None - elif condition_type == _ConditionType.Exists: - return left_operand is not None and left_operand != "" and left_operand != [] - elif condition_type == _ConditionType.NotExists: - return left_operand is None or left_operand == "" or left_operand == [] - else: - raise ValueError(f"Unsupported condition type: {condition_type}") - - def process(self, leftOperand: Any, rightOperand: Any): - result = self._evaluate(leftOperand, rightOperand) - - if result: - self.send("true", leftOperand) - else: - self.send("false", leftOperand) - - -class GetAttribute(Component): - """Get an attribute from an object or dictionary.""" - - inputs = { - "object": Input(description="The object or dictionary"), - "key": Input(description="The attribute name", type=str), - } - outputs = {"value": Output(description="The attribute value")} - - def __init__(self, macro_data: dict, **kwargs): - super().__init__(**kwargs) - if "key" not in macro_data: - raise ValueError("Missing 'key' in GetAttribute configuration.") - key = macro_data["key"] - self.value = None - if "value" in key: - self.value = key["value"] - if "type" in key: - if key["type"] == "static": - self.inputs["key"]._input_mode = InputMode.STATIC # type: ignore - self.inputs["key"].value = self.value - else: - self.inputs["key"]._input_mode = InputMode.STICKY # type: ignore - if self.value is not None: - self.inputs["key"].value = self.value - - def process(self, object: Any, key: str): - keys = key.split(".") - value = object - for k in keys: - if isinstance(value, dict): - value = value.get(k, None) - elif hasattr(value, k): - value = getattr(value, k) - else: - value = None - break - self.send("value", value) diff --git a/flyde/stdlib.pyi b/flyde/stdlib.pyi deleted file mode 100644 index fe02ae1..0000000 --- a/flyde/stdlib.pyi +++ /dev/null @@ -1,52 +0,0 @@ -from _typeshed import Incomplete -from enum import Enum -from flyde.io import Input as Input, InputMode as InputMode, Output as Output -from flyde.node import Component as Component -from typing import Any - -class InlineValue(Component): - """InlineValue sends a constant value to output.""" - outputs: Incomplete - value: Incomplete - def __init__(self, macro_data: dict, **kwargs) -> None: ... - def process(self) -> None: ... - def _is_inline_dict(self, value: Any) -> bool: - """Check if a value is an inline Flyde value dict, which has `type` and `value` keys.""" - def _get_inline_value(self, value: Any) -> Any: - """Get the value from an inline Flyde value output.""" - -class _ConditionType(Enum): - """Condition type enumeration.""" - Equal = 'EQUAL' - NotEqual = 'NOT_EQUAL' - Contains = 'CONTAINS' - NotContains = 'NOT_CONTAINS' - RegexMatches = 'REGEX_MATCHES' - Exists = 'EXISTS' - NotExists = 'NOT_EXISTS' - -class _ConditionalConfig: - """Conditional configuration.""" - property_path: Incomplete - condition_type: Incomplete - condition_data: Incomplete - left_operand: Incomplete - right_operand: Incomplete - def __init__(self, yml: dict) -> None: ... - -class Conditional(Component): - """Conditional component evaluates a condition against the input and sends the result to output.""" - inputs: Incomplete - outputs: Incomplete - _config: Incomplete - def __init__(self, macro_data: dict, **kwargs) -> None: ... - def _evaluate(self, left_operand: Any, right_operand: Any) -> bool: ... - def process(self, leftOperand: Any, rightOperand: Any): ... - -class GetAttribute(Component): - """Get an attribute from an object or dictionary.""" - inputs: Incomplete - outputs: Incomplete - value: Incomplete - def __init__(self, macro_data: dict, **kwargs) -> None: ... - def process(self, object: Any, key: str): ... diff --git a/pyproject.toml b/pyproject.toml index e820517..72ef2c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pyflyde" -version = "0.0.12" +version = "0.1.0" requires-python = ">= 3.9" authors = [{ name = "Vladimir Sibirov" }] description = "Python SDK and runtime for Flyde - a visual flow-based programming language and IDE." @@ -18,6 +18,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Operating System :: OS Independent", "Topic :: Scientific/Engineering", "Topic :: Software Development :: Libraries", @@ -35,6 +36,9 @@ pyflyde = "flyde.cli:main" # package-dir = { "" = "flyde" } packages = ["flyde"] +[tool.setuptools.package-data] +flyde = ["py.typed"] + [project.optional-dependencies] dev = [ "setuptools", @@ -45,4 +49,5 @@ dev = [ "mypy", "twine", "mkdocs", + "icecream", ] diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..f11cf63 --- /dev/null +++ b/ruff.toml @@ -0,0 +1 @@ +line-length = 120 diff --git a/tests/Repeat3Times.flyde b/tests/Repeat3Times.flyde index 91c84ee..0a1dea9 100644 --- a/tests/Repeat3Times.flyde +++ b/tests/Repeat3Times.flyde @@ -1,37 +1,53 @@ -imports: - "@flyde/stdlib": - - InlineValue - components.flyde.ts: - - RepeatWordNTimes - - Capitalize +imports: {} node: instances: - pos: x: -105.91999999999999 - y: 47.08 + y: 49.08 id: RepeatWordNTimes-h30493h inputConfig: times: mode: sticky nodeId: RepeatWordNTimes + config: + word: + type: dynamic + value: "{{word}}" + times: + type: dynamic + value: "{{times}}" + type: code + source: + type: custom + data: custom://components.py/RepeatWordNTimes - pos: - x: 13.582794189453125 - y: -132.9213885498047 + x: -422.4172058105469 + y: 126.0786114501953 id: inl-ytsuyrje4syeb4qduymsfkl2 inputConfig: {} - visibleInputs: [] - nodeId: InlineValue__inl-ytsuyrje4syeb4qduymsfkl2 - macroId: InlineValue - macroData: + nodeId: InlineValue + config: value: type: number value: 3 + type: code + source: + type: package + data: "@flyde/nodes" - pos: - x: -68.75665283203125 - y: 194.59005737304688 + x: 109.24334716796875 + y: 55.590057373046875 id: Capitalize-790499u inputConfig: {} nodeId: Capitalize + config: + inp: + type: dynamic + value: "{{inp}}" + type: code + source: + type: custom + data: custom://components.py/Capitalize connections: - from: insId: inl-ytsuyrje4syeb4qduymsfkl2 @@ -66,13 +82,13 @@ node: delayed: false inputsPosition: word: - x: -92.08137329101562 - y: -227.9133740234375 + x: -351.0813732910156 + y: 52.08662597656249 outputsPosition: result: x: -23.264428942324532 y: 237.25953921502617 word3x: - x: -40.05281249999999 - y: 369.2831286621094 - description: For each input string, sends a string with the same conent repeated 3 times + x: 345.94718750000004 + y: 57.28312866210939 + description: For each input string, sends a string with the same content repeated 3 times diff --git a/tests/TestCustomLoad.flyde b/tests/TestCustomLoad.flyde new file mode 100644 index 0000000..3592879 --- /dev/null +++ b/tests/TestCustomLoad.flyde @@ -0,0 +1,66 @@ +imports: {} +node: + instances: + - pos: + x: -200 + y: 50 + id: custom-echo-test + inputConfig: {} + nodeId: Echo + config: + inp: + type: dynamic + value: "{{inp}}" + type: code + source: + type: custom + data: custom://components.py/Echo + - pos: + x: 50 + y: 50 + id: custom-capitalize-test + inputConfig: {} + nodeId: Capitalize + config: + inp: + type: dynamic + value: "{{inp}}" + type: code + source: + type: custom + data: custom://components.py/Capitalize + connections: + - from: + insId: __this + pinId: input + to: + insId: custom-echo-test + pinId: inp + - from: + insId: custom-echo-test + pinId: out + to: + insId: custom-capitalize-test + pinId: inp + - from: + insId: custom-capitalize-test + pinId: out + to: + insId: __this + pinId: output + id: TestCustomLoad + inputs: + input: + mode: required + outputs: + output: + delayed: false + inputsPosition: + input: + x: -350 + y: 50 + outputsPosition: + output: + x: 200 + y: 50 + description: Test flow that loads custom components using custom:// source format diff --git a/tests/TestFanIn.flyde b/tests/TestFanIn.flyde index 25f9cad..25bcbd0 100644 --- a/tests/TestFanIn.flyde +++ b/tests/TestFanIn.flyde @@ -1,10 +1,4 @@ -imports: - "@flyde/stdlib": - - InlineValue - components.flyde.ts: - - Format - - Capitalize - - Echo +imports: {} node: instances: - pos: @@ -13,32 +7,62 @@ node: id: Format-4s04bag inputConfig: {} nodeId: Format + config: + inp: + type: dynamic + value: "{{inp}}" + format: + type: dynamic + value: "{{format}}" + type: code + source: + type: custom + data: custom://components.py/Format - pos: - x: -64.1663818359375 - y: 56.65972900390625 + x: -276.1663818359375 + y: -58.34027099609375 id: Capitalize-ch04buf inputConfig: {} nodeId: Capitalize + config: + inp: + type: dynamic + value: "{{inp}}" + type: code + source: + type: custom + data: custom://components.py/Capitalize - pos: - x: -148.65975585937497 - y: 238.9669189453125 + x: -5.659755859374968 + y: 54.9669189453125 id: Echo-eb14b06 inputConfig: {} nodeId: Echo + config: + inp: + type: dynamic + value: "{{inp}}" + type: code + source: + type: custom + data: custom://components.py/Echo - pos: - x: -328.1567211914063 - y: -90.1646710205078 + x: -594.1567211914063 + y: 120.8353289794922 id: xzi4aah4ewf1iw7q4a3jz9oj inputConfig: {} - nodeId: InlineValue__xzi4aah4ewf1iw7q4a3jz9oj - macroId: InlineValue - macroData: + nodeId: InlineValue + config: type: type: string value: string value: type: string value: Hello, {inp}! + type: code + source: + type: package + data: "@flyde/nodes" connections: - from: insId: Format-4s04bag @@ -82,7 +106,7 @@ node: to: insId: __this pinId: out - id: Example + id: TestFanIn inputs: str: mode: required @@ -91,12 +115,12 @@ node: delayed: false inputsPosition: str: - x: -119.99874877929688 - y: -99.46359436035156 + x: -606.9987487792969 + y: -29.463594360351564 outputsPosition: result: x: -23.264428942324532 y: 237.25953921502617 out: - x: -120.79533203125001 - y: 390.6666717529297 + x: 223.20466796875 + y: 54.66667175292969 diff --git a/tests/TestFanInGraph.flyde b/tests/TestFanInGraph.flyde index 4b191a7..fe50aad 100644 --- a/tests/TestFanInGraph.flyde +++ b/tests/TestFanInGraph.flyde @@ -1,11 +1,4 @@ -imports: - "@flyde/stdlib": - - InlineValue - components.flyde.ts: - - Format - - Capitalize - Repeat3Times.flyde: - - Repeat3Times +imports: {} node: instances: - pos: @@ -14,32 +7,58 @@ node: id: Format-4s04bag inputConfig: {} nodeId: Format + config: + inp: + type: dynamic + value: "{{inp}}" + format: + type: dynamic + value: "{{format}}" + type: code + source: + type: custom + data: custom://components.py/Format - pos: - x: -44.16827392578125 + x: -58.16827392578125 y: 63.02734375 id: Capitalize-ch04buf inputConfig: {} nodeId: Capitalize + config: + inp: + type: dynamic + value: "{{inp}}" + type: code + source: + type: custom + data: custom://components.py/Capitalize - pos: - x: -328.1567211914063 - y: -90.1646710205078 + x: -601.1567211914063 + y: 121.8353289794922 id: xzi4aah4ewf1iw7q4a3jz9oj inputConfig: {} - nodeId: InlineValue__xzi4aah4ewf1iw7q4a3jz9oj - macroId: InlineValue - macroData: + nodeId: InlineValue + config: type: type: string value: string value: type: string value: Hello, {inp}! + type: code + source: + type: package + data: "@flyde/nodes" - pos: - x: -185.60189453125 - y: 240.50311500933114 + x: 187.39810546875 + y: 119.50311500933117 id: Repeat3Times-ou04byd inputConfig: {} nodeId: Repeat3Times + type: visual + source: + type: file + data: Repeat3Times.flyde connections: - from: insId: __this @@ -83,7 +102,7 @@ node: to: insId: __this pinId: out - id: Example + id: TestFanInGraph inputs: str: mode: required @@ -92,12 +111,12 @@ node: delayed: false inputsPosition: str: - x: -119.99874877929688 - y: -99.46359436035156 + x: -593.9987487792969 + y: -21.463594360351564 outputsPosition: result: x: -23.264428942324532 y: 237.25953921502617 out: - x: -120.79533203125001 - y: 390.6666717529297 + x: 396.20466796874996 + y: 119.66667175292969 diff --git a/tests/TestInOutFlow.flyde b/tests/TestInOutFlow.flyde index 0e55838..d463964 100644 --- a/tests/TestInOutFlow.flyde +++ b/tests/TestInOutFlow.flyde @@ -1,26 +1,27 @@ -imports: - "@flyde/stdlib": - - Conditional - - InlineValue - components.flyde.ts: - - Echo - - Format +imports: {} node: instances: - pos: - x: -79.91999999999999 - y: 47.5 + x: 244.08 + y: -87.5 id: Echo-h3049mb inputConfig: {} nodeId: Echo + config: + inp: + type: dynamic + value: "{{inp}}" + type: code + source: + type: custom + data: custom://components.py/Echo - pos: x: -17.077154541015602 y: -87.20937501999856 id: ppsa1z6ja2w6yyo0sig7hvww inputConfig: {} - nodeId: Conditional__ppsa1z6ja2w6yyo0sig7hvww - macroId: Conditional - macroData: + nodeId: Conditional + config: leftOperand: type: dynamic value: "{{value}}" @@ -29,20 +30,34 @@ node: type: string condition: type: EXISTS + type: code + source: + type: package + data: "@flyde/nodes" - pos: - x: 96.42891601562502 - y: 48.46386716750146 + x: 241.42891601562496 + y: 13.463867167501462 id: Format-ve0397r inputConfig: {} nodeId: Format + config: + inp: + type: dynamic + value: "{{inp}}" + format: + type: dynamic + value: "{{format}}" + type: code + source: + type: custom + data: custom://components.py/Format - pos: - x: 227.21648071289064 - y: -86.37317262742044 + x: -32.78351928710936 + y: 33.626827372579555 id: apqbu37rhnui31o8qaud8ek8 inputConfig: {} - nodeId: InlineValue__apqbu37rhnui31o8qaud8ek8 - macroId: InlineValue - macroData: + nodeId: InlineValue + config: type: type: string value: string @@ -52,6 +67,10 @@ node: label: type: string value: '"ERR: msg is empty"' + type: code + source: + type: package + data: "@flyde/nodes" connections: - from: insId: Echo-h3049mb @@ -89,7 +108,7 @@ node: to: insId: Format-ve0397r pinId: format - id: Example + id: TestInOutFlow inputs: inMsg: mode: required @@ -98,12 +117,12 @@ node: delayed: false inputsPosition: inMsg: - x: 35.251495361328125 - y: -151.84883300781252 + x: -184.74850463867188 + y: -75.84883300781252 outputsPosition: result: x: -23.264428942324532 y: 237.25953921502617 outMsg: - x: -70.891923828125 - y: 220.63164794921875 + x: 631.1080761718749 + y: -93.36835205078125 diff --git a/tests/TestIsolatedFlow.flyde b/tests/TestIsolatedFlow.flyde index b856618..971b44a 100644 --- a/tests/TestIsolatedFlow.flyde +++ b/tests/TestIsolatedFlow.flyde @@ -1,27 +1,34 @@ -imports: - "@flyde/stdlib": - - InlineValue - components.flyde.ts: - - Echo +imports: {} node: instances: - pos: - x: -95.34318359375001 - y: -154.18736450195314 + x: -336.34318359375004 + y: 59.81263549804686 id: qxavnllf1foh9ivxvtz2hkad inputConfig: {} - nodeId: InlineValue__qxavnllf1foh9ivxvtz2hkad - macroId: InlineValue - macroData: + nodeId: InlineValue + config: value: type: string value: Hello + type: code + source: + type: package + data: "@flyde/nodes" - pos: x: -83.18318359374999 y: 60.36263549804687 id: Echo-x8049tw inputConfig: {} nodeId: Echo + config: + inp: + type: dynamic + value: "{{inp}}" + type: code + source: + type: custom + data: custom://components.py/Echo connections: - from: insId: qxavnllf1foh9ivxvtz2hkad @@ -29,7 +36,7 @@ node: to: insId: Echo-x8049tw pinId: inp - id: Example + id: TestIsolatedFlow inputs: {} outputs: {} inputsPosition: {} diff --git a/tests/TestNestedFlow.flyde b/tests/TestNestedFlow.flyde index 56e3701..921865d 100644 --- a/tests/TestNestedFlow.flyde +++ b/tests/TestNestedFlow.flyde @@ -1,24 +1,20 @@ -imports: - "@flyde/stdlib": [] - Repeat3Times.flyde: - - Repeat3Times - components.flyde.ts: - - Echo - - RepeatWordNTimes +imports: {} node: instances: - pos: - x: -234.30030517578126 - y: -38.32338623046875 - id: Repeat3Times-u6049uf - inputConfig: {} - nodeId: Repeat3Times - - pos: - x: -196.171416015625 - y: 107.68781005859375 + x: -465.54763671875 + y: 195.6628924560547 id: Echo-co1498h inputConfig: {} nodeId: Echo + config: + inp: + type: dynamic + value: "{{inp}}" + type: code + source: + type: custom + data: custom://components.py/Echo - pos: x: -195.52671508789064 y: 269.8743734741211 @@ -27,19 +23,28 @@ node: times: mode: sticky nodeId: RepeatWordNTimes + config: + word: + type: dynamic + value: "{{word}}" + times: + type: dynamic + value: "{{times}}" + type: code + source: + type: custom + data: custom://components.py/RepeatWordNTimes + - pos: + x: -674.5008868408204 + y: 195.91538436889653 + id: Repeat3Times-dfk3brg0 + inputConfig: {} + nodeId: Repeat3Times + type: visual + source: + type: file + data: Repeat3Times.flyde connections: - - from: - insId: __this - pinId: inp - to: - insId: Repeat3Times-u6049uf - pinId: word - - from: - insId: Repeat3Times-u6049uf - pinId: word3x - to: - insId: Echo-co1498h - pinId: inp - from: insId: Echo-co1498h pinId: out @@ -58,6 +63,18 @@ node: to: insId: RepeatWordNTimes-pp249oy pinId: times + - from: + insId: Repeat3Times-dfk3brg0 + pinId: word3x + to: + insId: Echo-co1498h + pinId: inp + - from: + insId: __this + pinId: inp + to: + insId: Repeat3Times-dfk3brg0 + pinId: word id: TestNestedFlow inputs: inp: @@ -69,19 +86,19 @@ node: delayed: false inputsPosition: inp: - x: -172.50668701171875 - y: -161.04999633789063 + x: -854.638095703125 + y: 195.21616271972655 x: x: 37.526656494140624 y: -159.08574951171875 n: - x: 37.526656494140624 - y: -159.08574951171875 + x: -448.31001342773436 + y: 288.3494387817383 outputsPosition: result: x: -23.264428942324532 y: 237.25953921502617 out: - x: -110.85763549804688 - y: 444.78157318115234 + x: 20.572113037109375 + y: 280.7653988647461 description: Repeats input 3xN times diff --git a/tests/TestStdlib.flyde b/tests/TestStdlib.flyde new file mode 100644 index 0000000..9b88c04 --- /dev/null +++ b/tests/TestStdlib.flyde @@ -0,0 +1,51 @@ +imports: {} +node: + instances: + - pos: + x: -150 + y: 20 + id: Http-dd04gea + inputConfig: {} + nodeId: Http + config: + method: + type: select + value: GET + url: + type: string + value: https://www.flyde.dev + data: + type: json + value: {} + params: + type: json + value: + ref: pyflyde + headers: + type: json + value: + User-Agent: PyFlyde v0.1 + type: code + source: + type: package + data: "@flyde/nodes" + connections: + - from: + insId: Http-dd04gea + pinId: data + to: + insId: __this + pinId: response + id: TestStdlib + inputs: {} + outputs: + response: + delayed: false + inputsPosition: {} + outputsPosition: + result: + x: -23.264428942324532 + y: 237.25953921502617 + response: + x: 124.5 + y: 19 diff --git a/tests/__init.py__ b/tests/__init__.py similarity index 100% rename from tests/__init.py__ rename to tests/__init__.py diff --git a/tests/components.flyde.ts b/tests/components.flyde.ts deleted file mode 100644 index 1b38c90..0000000 --- a/tests/components.flyde.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { CodeNode } from "@flyde/core"; - -export const Format: CodeNode = { - id: "Format", - description: "Formats the input value with a given format string and sends it to out.", - inputs: { - inp: { description: "The input" }, - format: { description: "The format string" } - }, - outputs: { - out: { description: "The formatted output" } - }, - run: () => { return; }, -}; - -export const Echo: CodeNode = { - id: "Echo", - description: "A simple component that echoes the input.", - inputs: { - inp: { description: "The input" } - }, - outputs: { - out: { description: "The output" } - }, - run: () => { return; }, -}; - -export const Capitalize: CodeNode = { - id: "Capitalize", - description: "A component that capitalizes the input.", - inputs: { - inp: { description: "The input" } - }, - outputs: { - out: { description: "The output" } - }, - run: () => { return; }, -}; - -export const RepeatWordNTimes: CodeNode = { - id: "RepeatWordNTimes", - description: "A component that has both inputs and outputs and a sticky input.", - inputs: { - word: { description: "The input" }, - times: { description: "The number of times to repeat the input" } - }, - outputs: { - out: { description: "The output" } - }, - run: () => { return; }, -}; - diff --git a/tests/components.py b/tests/components.py index 2e91cf7..4609042 100644 --- a/tests/components.py +++ b/tests/components.py @@ -7,9 +7,7 @@ class Format(Component): inputs = { "inp": Input(description="The input"), - "format": Input( - description="The format string", type=str, mode=InputMode.STICKY - ), + "format": Input(description="The format string", type=str, mode=InputMode.STICKY), } outputs = { "out": Output(description="The formatted output", type=str), diff --git a/tests/flyde-nodes.json b/tests/flyde-nodes.json new file mode 100644 index 0000000..bcb21aa --- /dev/null +++ b/tests/flyde-nodes.json @@ -0,0 +1,631 @@ +{ + "nodes": { + "Capitalize": { + "id": "Capitalize", + "type": "code", + "displayName": "Capitalize", + "description": "A component that capitalizes the input.", + "icon": "fa-brands fa-python", + "source": { + "type": "custom", + "data": "custom://components.py/Capitalize" + }, + "editorNode": { + "id": "Capitalize", + "displayName": "Capitalize", + "description": "A component that capitalizes the input.", + "inputs": { + "inp": { + "description": "The input" + } + }, + "outputs": { + "out": { + "description": "The output" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "Echo": { + "id": "Echo", + "type": "code", + "displayName": "Echo", + "description": "A simple component that echoes the input.", + "icon": "fa-brands fa-python", + "source": { + "type": "custom", + "data": "custom://components.py/Echo" + }, + "editorNode": { + "id": "Echo", + "displayName": "Echo", + "description": "A simple component that echoes the input.", + "inputs": { + "inp": { + "description": "The input" + } + }, + "outputs": { + "out": { + "description": "The output" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "Format": { + "id": "Format", + "type": "code", + "displayName": "Format", + "description": "Formats the input value with a given format string and sends it to out.", + "icon": "fa-brands fa-python", + "source": { + "type": "custom", + "data": "custom://components.py/Format" + }, + "editorNode": { + "id": "Format", + "displayName": "Format", + "description": "Formats the input value with a given format string and sends it to out.", + "inputs": { + "inp": { + "description": "The input" + }, + "format": { + "description": "The format string" + } + }, + "outputs": { + "out": { + "description": "The formatted output" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "RepeatWordNTimes": { + "id": "RepeatWordNTimes", + "type": "code", + "displayName": "Repeat Word NTimes", + "description": "A component that has both inputs and outputs and a sticky input.", + "icon": "fa-brands fa-python", + "source": { + "type": "custom", + "data": "custom://components.py/RepeatWordNTimes" + }, + "editorNode": { + "id": "RepeatWordNTimes", + "displayName": "Repeat Word NTimes", + "description": "A component that has both inputs and outputs and a sticky input.", + "inputs": { + "word": { + "description": "The input" + }, + "times": { + "description": "The number of times to repeat the input" + } + }, + "outputs": { + "out": { + "description": "The output" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "CustomBob": { + "id": "CustomBob", + "type": "code", + "displayName": "Custom Bob", + "description": "A custom external node named Bob", + "icon": "fa-brands fa-python", + "source": { + "type": "custom", + "data": "custom://test_cli.py/CustomBob" + }, + "editorNode": { + "id": "CustomBob", + "displayName": "Custom Bob", + "description": "A custom external node named Bob", + "inputs": { + "value": { + "description": "Input value to process" + } + }, + "outputs": { + "result": { + "description": "Processed result from external runtime" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "InlineValue": "@flyde/nodes", + "TestCustomComponent": { + "id": "TestCustomComponent", + "type": "code", + "displayName": "Test Custom Component", + "description": "A test component for unit testing.", + "icon": "fa-brands fa-python", + "source": { + "type": "custom", + "data": "custom://test_cli.py/TestCustomComponent" + }, + "editorNode": { + "id": "TestCustomComponent", + "displayName": "Test Custom Component", + "description": "A test component for unit testing.", + "inputs": { + "input1": { + "description": "First test input" + }, + "input2": { + "description": "Second test input" + } + }, + "outputs": { + "output": { + "description": "Test output" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "AllStickyInputsComponent": { + "id": "AllStickyInputsComponent", + "type": "code", + "displayName": "All Sticky Inputs Component", + "description": "A component with only sticky inputs to test single execution.", + "icon": "fa-brands fa-python", + "source": { + "type": "custom", + "data": "custom://test_node.py/AllStickyInputsComponent" + }, + "editorNode": { + "id": "AllStickyInputsComponent", + "displayName": "All Sticky Inputs Component", + "description": "A component with only sticky inputs to test single execution.", + "inputs": { + "a": { + "description": "First sticky input" + }, + "b": { + "description": "Second sticky input" + } + }, + "outputs": { + "result": { + "description": "Result of operation" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "CustomRunComponent": { + "id": "CustomRunComponent", + "type": "code", + "displayName": "Custom Run Component", + "description": "A component that has a custom run and shutdown handlers.", + "icon": "fa-brands fa-python", + "source": { + "type": "custom", + "data": "custom://test_node.py/CustomRunComponent" + }, + "editorNode": { + "id": "CustomRunComponent", + "displayName": "Custom Run Component", + "description": "A component that has a custom run and shutdown handlers.", + "inputs": { + "s": { + "description": "Individual strings" + } + }, + "outputs": { + "l": { + "description": "List of strings" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "InvalidSendProcess": { + "id": "InvalidSendProcess", + "type": "code", + "displayName": "Invalid Send Process", + "description": "A component that tries to send a message without a corresponding output.", + "icon": "fa-brands fa-python", + "source": { + "type": "custom", + "data": "custom://test_node.py/InvalidSendProcess" + }, + "editorNode": { + "id": "InvalidSendProcess", + "displayName": "Invalid Send Process", + "description": "A component that tries to send a message without a corresponding output.", + "inputs": { + "s": { + "description": "Individual strings" + } + }, + "outputs": {}, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "NoProcessComponent": { + "id": "NoProcessComponent", + "type": "code", + "displayName": "No Process Component", + "description": "A component to test no inputs, outputs, and no process method.", + "icon": "fa-brands fa-python", + "source": { + "type": "custom", + "data": "custom://test_node.py/NoProcessComponent" + }, + "editorNode": { + "id": "NoProcessComponent", + "displayName": "No Process Component", + "description": "A component to test no inputs, outputs, and no process method.", + "inputs": {}, + "outputs": {}, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "SinkComponent": { + "id": "SinkComponent", + "type": "code", + "displayName": "Sink Component", + "description": "A component that only has inputs.", + "icon": "fa-brands fa-python", + "source": { + "type": "custom", + "data": "custom://test_node.py/SinkComponent" + }, + "editorNode": { + "id": "SinkComponent", + "displayName": "Sink Component", + "description": "A component that only has inputs.", + "inputs": { + "word": { + "description": "The input" + }, + "output": { + "description": "Object to store result in" + } + }, + "outputs": {}, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "SourceComponent": { + "id": "SourceComponent", + "type": "code", + "displayName": "Source Component", + "description": "A component that only has outputs.", + "icon": "fa-brands fa-python", + "source": { + "type": "custom", + "data": "custom://test_node.py/SourceComponent" + }, + "editorNode": { + "id": "SourceComponent", + "displayName": "Source Component", + "description": "A component that only has outputs.", + "inputs": {}, + "outputs": { + "out": { + "description": "The output" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "TestStdlib": { + "id": "TestStdlib", + "type": "visual", + "displayName": "Test Stdlib", + "description": "", + "icon": "fa-diagram-project", + "source": { + "type": "file", + "data": "TestStdlib.flyde" + }, + "editorNode": { + "id": "TestStdlib", + "displayName": "Test Stdlib", + "description": "", + "inputs": {}, + "outputs": { + "response": { + "description": "response output" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "Repeat3Times": { + "id": "Repeat3Times", + "type": "visual", + "displayName": "Repeat3Times", + "description": "For each input string, sends a string with the same content repeated 3 times", + "icon": "fa-diagram-project", + "source": { + "type": "file", + "data": "Repeat3Times.flyde" + }, + "editorNode": { + "id": "Repeat3Times", + "displayName": "Repeat3Times", + "description": "For each input string, sends a string with the same content repeated 3 times", + "inputs": { + "word": { + "description": "word input (required)" + } + }, + "outputs": { + "word3x": { + "description": "word3x output" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "TestNestedFlow": { + "id": "TestNestedFlow", + "type": "visual", + "displayName": "Test Nested Flow", + "description": "Repeats input 3xN times", + "icon": "fa-diagram-project", + "source": { + "type": "file", + "data": "TestNestedFlow.flyde" + }, + "editorNode": { + "id": "TestNestedFlow", + "displayName": "Test Nested Flow", + "description": "Repeats input 3xN times", + "inputs": { + "inp": { + "description": "inp input (required)" + }, + "n": { + "description": "n input (required)" + } + }, + "outputs": { + "out": { + "description": "out output" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "TestFanInGraph": { + "id": "TestFanInGraph", + "type": "visual", + "displayName": "Test Fan In Graph", + "description": "", + "icon": "fa-diagram-project", + "source": { + "type": "file", + "data": "TestFanInGraph.flyde" + }, + "editorNode": { + "id": "TestFanInGraph", + "displayName": "Test Fan In Graph", + "description": "", + "inputs": { + "str": { + "description": "str input (required)" + } + }, + "outputs": { + "out": { + "description": "out output" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "TestFanIn": { + "id": "TestFanIn", + "type": "visual", + "displayName": "Test Fan In", + "description": "", + "icon": "fa-diagram-project", + "source": { + "type": "file", + "data": "TestFanIn.flyde" + }, + "editorNode": { + "id": "TestFanIn", + "displayName": "Test Fan In", + "description": "", + "inputs": { + "str": { + "description": "str input (required)" + } + }, + "outputs": { + "out": { + "description": "out output" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "TestIsolatedFlow": { + "id": "TestIsolatedFlow", + "type": "visual", + "displayName": "Test Isolated Flow", + "description": "", + "icon": "fa-diagram-project", + "source": { + "type": "file", + "data": "TestIsolatedFlow.flyde" + }, + "editorNode": { + "id": "TestIsolatedFlow", + "displayName": "Test Isolated Flow", + "description": "", + "inputs": {}, + "outputs": {}, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "TestCustomLoad": { + "id": "TestCustomLoad", + "type": "visual", + "displayName": "Test Custom Load", + "description": "Test flow that loads custom components using custom:// source format", + "icon": "fa-diagram-project", + "source": { + "type": "file", + "data": "TestCustomLoad.flyde" + }, + "editorNode": { + "id": "TestCustomLoad", + "displayName": "Test Custom Load", + "description": "Test flow that loads custom components using custom:// source format", + "inputs": { + "input": { + "description": "input input (required)" + } + }, + "outputs": { + "output": { + "description": "output output" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "TestInOutFlow": { + "id": "TestInOutFlow", + "type": "visual", + "displayName": "Test In Out Flow", + "description": "", + "icon": "fa-diagram-project", + "source": { + "type": "file", + "data": "TestInOutFlow.flyde" + }, + "editorNode": { + "id": "TestInOutFlow", + "displayName": "Test In Out Flow", + "description": "", + "inputs": { + "inMsg": { + "description": "inMsg input (required)" + } + }, + "outputs": { + "outMsg": { + "description": "outMsg output" + } + }, + "editorConfig": { + "type": "structured" + } + }, + "config": {} + }, + "Conditional": "@flyde/nodes", + "GetAttribute": "@flyde/nodes", + "Http": "@flyde/nodes" + }, + "groups": [ + { + "title": "Your PyFlyde Nodes", + "nodeIds": [ + "Capitalize", + "Echo", + "Format", + "RepeatWordNTimes", + "CustomBob", + "InlineValue", + "TestCustomComponent", + "AllStickyInputsComponent", + "CustomRunComponent", + "InvalidSendProcess", + "NoProcessComponent", + "SinkComponent", + "SourceComponent", + "TestStdlib", + "Repeat3Times", + "TestNestedFlow", + "TestFanInGraph", + "TestFanIn", + "TestIsolatedFlow", + "TestCustomLoad", + "TestInOutFlow" + ] + }, + { + "title": "PyFlyde Standard Nodes", + "nodeIds": [ + "Conditional", + "GetAttribute", + "Http", + "InlineValue" + ] + } + ] +} \ No newline at end of file diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..6ca1b3a --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,659 @@ +import json +import os +import shutil +import tempfile +import unittest + +from flyde.cli import ( + collect_components_from_directory, + collect_flyde_nodes_from_directory, + convert_class_name_to_display_name, + gen_json, + generate_flyde_node_json, + generate_node_json, + is_stdlib_node, +) +from flyde.io import Input, Output +from flyde.node import Component +from flyde.nodes import list_nodes + + +class TestCLIHelpers(unittest.TestCase): + def test_convert_class_name_to_display_name(self): + test_cases = [ + ("MyCustomNode", "My Custom Node"), + ("HTTPClient", "HTTPClient"), # Adjacent capitals stay together + ("XMLParser", "XMLParser"), + ("SimpleNode", "Simple Node"), + ("Node", "Node"), # Single word + ("MyHTTPClient", "My HTTPClient"), + ("CustomBob", "Custom Bob"), + ("CustomAlice", "Custom Alice"), + ] + + for class_name, expected in test_cases: + with self.subTest(class_name=class_name): + result = convert_class_name_to_display_name(class_name) + self.assertEqual(result, expected) + + def test_is_stdlib_node(self): + # Test that all supported macros are detected as stdlib nodes + for macro in list_nodes(): + with self.subTest(node_name=macro): + result = is_stdlib_node(macro) + self.assertTrue(result) + + # Test that custom nodes are not detected as stdlib nodes + custom_nodes = ["CustomNode", "MyComponent", "Echo", "Format"] + for node_name in custom_nodes: + with self.subTest(node_name=node_name): + result = is_stdlib_node(node_name) + self.assertFalse(result) + + +class TestCustomComponent(Component): + """A test component for unit testing.""" + + inputs = { + "input1": Input(description="First test input"), + "input2": Input(description="Second test input"), + } + outputs = { + "output": Output(description="Test output"), + } + + def process(self, input1, input2): + return {"output": f"{input1}-{input2}"} + + +class CustomBob(Component): + """A custom external node named Bob""" + + inputs = { + "value": Input(description="Input value to process"), + } + outputs = { + "result": Output(description="Processed result from external runtime"), + } + + def process(self, value): + return {"result": f"Bob processed: {value}"} + + +class InlineValue(Component): + """This overrides the standard InlineValue node""" + + outputs = { + "value": Output(description="The overridden value"), + } + + def process(self): + return {"value": "overridden"} + + +class TestGenerateNodeJson(unittest.TestCase): + def test_generate_custom_node_json(self): + result = generate_node_json("CustomBob", CustomBob, "test_components.py") + + assert isinstance(result, dict) + expected = { + "id": "CustomBob", + "type": "code", + "displayName": "Custom Bob", + "description": "A custom external node named Bob", + "icon": "fa-brands fa-python", + "source": { + "type": "custom", + "data": "custom://test_components.py/CustomBob", + }, + "editorNode": { + "id": "CustomBob", + "displayName": "Custom Bob", + "description": "A custom external node named Bob", + "inputs": {"value": {"description": "Input value to process"}}, + "outputs": {"result": {"description": "Processed result from external runtime"}}, + "editorConfig": {"type": "structured"}, + }, + "config": {}, + } + + self.assertEqual(result, expected) + + def test_generate_stdlib_node_json(self): + result = generate_node_json("InlineValue", InlineValue, "test_components.py") + expected = "@flyde/nodes" + self.assertEqual(result, expected) + + def test_generate_node_with_no_docstring(self): + class NodeWithoutDoc(Component): + inputs = {"inp": Input(description="Input")} + outputs = {"out": Output(description="Output")} + + result = generate_node_json("NodeWithoutDoc", NodeWithoutDoc, "test.py") + + assert isinstance(result, dict) + self.assertEqual(result["description"], "") + self.assertEqual(result["editorNode"]["description"], "") + + def test_generate_node_with_no_inputs_outputs(self): + class EmptyNode(Component): + """An empty node""" + + result = generate_node_json("EmptyNode", EmptyNode, "test.py") + + assert isinstance(result, dict) + self.assertEqual(result["editorNode"]["inputs"], {}) + self.assertEqual(result["editorNode"]["outputs"], {}) + + +class TestCollectComponents(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.temp_dir) + + def test_collect_components_from_directory(self): + # Create test Python files + test_component_py = ''' +from flyde.io import Input, Output +from flyde.node import Component + +class TestNode1(Component): + """First test node""" + inputs = {"inp": Input(description="Input")} + outputs = {"out": Output(description="Output")} + +class TestNode2(Component): + """Second test node""" + inputs = {"data": Input(description="Data input")} + outputs = {"result": Output(description="Result output")} + +# This should be ignored +class NotAComponent: + pass +''' + + another_component_py = ''' +from flyde.io import Input, Output +from flyde.node import Component + +class AnotherNode(Component): + """Another test node""" + outputs = {"value": Output(description="Value output")} +''' + + # Write test files + with open(os.path.join(self.temp_dir, "test_components.py"), "w") as f: + f.write(test_component_py) + + with open(os.path.join(self.temp_dir, "another.py"), "w") as f: + f.write(another_component_py) + + # Create __init__.py (should be ignored) + with open(os.path.join(self.temp_dir, "__init__.py"), "w") as f: + f.write("# This should be ignored") + + # Collect components + components = collect_components_from_directory(self.temp_dir) + + # Should find 3 components + self.assertEqual(len(components), 3) + self.assertIn("TestNode1", components) + self.assertIn("TestNode2", components) + self.assertIn("AnotherNode", components) + + # Check that components are actually Component subclasses + for name, component_info in components.items(): + self.assertTrue(issubclass(component_info["class"], Component)) + self.assertIn("file_path", component_info) + + def test_collect_components_invalid_syntax(self): + # Create a file with invalid Python syntax + invalid_py = """ +This is not valid Python syntax! +class InvalidNode(Component: + pass +""" + + with open(os.path.join(self.temp_dir, "invalid.py"), "w") as f: + f.write(invalid_py) + + # Should handle the error gracefully + components = collect_components_from_directory(self.temp_dir) + self.assertEqual(len(components), 0) + + def test_collect_components_empty_directory(self): + # Test with empty directory + components = collect_components_from_directory(self.temp_dir) + self.assertEqual(len(components), 0) + + +class TestCollectFlydeNodes(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.temp_dir) + + def test_collect_flyde_nodes_from_directory(self): + # Use existing test files from the tests directory + test_files_dir = os.path.join(os.path.dirname(__file__), "..") + flyde_nodes = collect_flyde_nodes_from_directory(os.path.join(test_files_dir, "tests")) + + # Should find several .flyde nodes + self.assertGreater(len(flyde_nodes), 0) + self.assertIn("TestNestedFlow", flyde_nodes) + self.assertIn("Repeat3Times", flyde_nodes) + + # Check TestNestedFlow structure (uses filename as ID) + nested_flow = flyde_nodes["TestNestedFlow"] + self.assertEqual(nested_flow["type"], "flyde") + self.assertEqual(nested_flow["description"], "Repeats input 3xN times") + self.assertIn("file_path", nested_flow) + self.assertEqual(len(nested_flow["inputs"]), 2) + self.assertEqual(len(nested_flow["outputs"]), 1) + self.assertEqual(nested_flow["inputs"]["inp"]["mode"], "required") + self.assertEqual(nested_flow["inputs"]["n"]["mode"], "required") + + # Check Repeat3Times structure + repeat_flow = flyde_nodes["Repeat3Times"] + self.assertEqual(repeat_flow["type"], "flyde") + self.assertIn("For each input string", repeat_flow["description"]) + self.assertEqual(len(repeat_flow["inputs"]), 1) + self.assertEqual(len(repeat_flow["outputs"]), 1) + + def test_collect_flyde_nodes_invalid_yaml(self): + # Create a file with invalid YAML syntax + invalid_flyde = """ +This is not valid YAML syntax! +node: + inputs: + - invalid structure +""" + + with open(os.path.join(self.temp_dir, "invalid.flyde"), "w") as f: + f.write(invalid_flyde) + + # Should handle the error gracefully + flyde_nodes = collect_flyde_nodes_from_directory(self.temp_dir) + self.assertEqual(len(flyde_nodes), 0) + + def test_collect_flyde_nodes_empty_directory(self): + # Test with empty directory + flyde_nodes = collect_flyde_nodes_from_directory(self.temp_dir) + self.assertEqual(len(flyde_nodes), 0) + + def test_collect_flyde_nodes_uses_filename_as_id(self): + # Test that node ID always comes from filename, not YAML id field + test_flyde = """imports: {} +node: + instances: [] + connections: [] + id: DummyExample + inputs: + inp: + mode: required + outputs: + out: + delayed: false + inputsPosition: {} + outputsPosition: {} +description: Test that uses filename for ID +""" + + with open(os.path.join(self.temp_dir, "ActualNodeName.flyde"), "w") as f: + f.write(test_flyde) + + flyde_nodes = collect_flyde_nodes_from_directory(self.temp_dir) + self.assertEqual(len(flyde_nodes), 1) + # Should use filename, not the "DummyExample" from YAML + self.assertIn("ActualNodeName", flyde_nodes) + self.assertNotIn("DummyExample", flyde_nodes) + + def test_collect_flyde_nodes_description_priority(self): + # Test that node.description takes priority over root description + test_flyde_node_desc = """imports: {} +node: + instances: [] + connections: [] + inputs: + inp: + mode: required + outputs: + out: + delayed: false + inputsPosition: {} + outputsPosition: {} + description: Node level description +description: Root level description +""" + + with open(os.path.join(self.temp_dir, "TestPriority.flyde"), "w") as f: + f.write(test_flyde_node_desc) + + flyde_nodes = collect_flyde_nodes_from_directory(self.temp_dir) + self.assertEqual(len(flyde_nodes), 1) + self.assertIn("TestPriority", flyde_nodes) + # Should prefer node.description over root description + self.assertEqual(flyde_nodes["TestPriority"]["description"], "Node level description") + + +class TestGenerateFlydeNodeJson(unittest.TestCase): + def test_generate_flyde_node_json(self): + self.maxDiff = None + flyde_info = { + "file_path": "test_flows/TestFlow.flyde", + "description": "A test flow for unit testing", + "inputs": {"input1": {"mode": "required"}, "input2": {"mode": "optional"}}, + "outputs": {"output1": {}, "output2": {}}, + "type": "flyde", + } + + result = generate_flyde_node_json("TestFlow", flyde_info) + + expected = { + "id": "TestFlow", + "type": "visual", + "displayName": "Test Flow", + "description": "A test flow for unit testing", + "icon": "fa-diagram-project", + "source": {"type": "file", "data": "test_flows/TestFlow.flyde"}, + "editorNode": { + "id": "TestFlow", + "displayName": "Test Flow", + "description": "A test flow for unit testing", + "inputs": { + "input1": {"description": "input1 input (required)"}, + "input2": {"description": "input2 input"}, + }, + "outputs": { + "output1": {"description": "output1 output"}, + "output2": {"description": "output2 output"}, + }, + "editorConfig": {"type": "structured"}, + }, + "config": {}, + } + + self.assertEqual(result, expected) + + def test_generate_flyde_node_with_no_inputs_outputs(self): + flyde_info = { + "file_path": "EmptyFlow.flyde", + "description": "An empty flow", + "inputs": {}, + "outputs": {}, + "type": "flyde", + } + + result = generate_flyde_node_json("EmptyFlow", flyde_info) + + self.assertEqual(result["editorNode"]["inputs"], {}) + self.assertEqual(result["editorNode"]["outputs"], {}) + + +class TestGenJson(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.temp_dir) + + def test_gen_json_with_mixed_components(self): + # Create test files with both custom and stdlib components, but do not override stdlib nodes + components_py = ''' +from flyde.io import Input, Output +from flyde.node import Component + +class CustomBob(Component): + """A custom external node named Bob""" + inputs = {"value": Input(description="Input value to process")} + outputs = {"result": Output(description="Processed result from external runtime")} + +class CustomAlice(Component): + """Another custom external node""" + inputs = { + "input1": Input(description="First input"), + "input2": Input(description="Second input") + } + outputs = {"output": Output(description="Combined output")} +''' + + with open(os.path.join(self.temp_dir, "components.py"), "w") as f: + f.write(components_py) + + # Generate JSON + gen_json(self.temp_dir) + + # Check that the file was created + output_file = os.path.join(self.temp_dir, "flyde-nodes.json") + self.assertTrue(os.path.exists(output_file)) + + # Load and verify the JSON content + with open(output_file, "r") as f: + data = json.load(f) + + # Check structure + self.assertIn("nodes", data) + self.assertIn("groups", data) + + # Check nodes + nodes = data["nodes"] + # Should have all custom nodes plus all stdlib nodes + expected_nodes = set(["CustomBob", "CustomAlice"] + list_nodes()) + self.assertEqual(set(nodes.keys()), expected_nodes) + self.assertIn("CustomBob", nodes) + self.assertIn("CustomAlice", nodes) + for stdlib_node in list_nodes(): + self.assertIn(stdlib_node, nodes) + + # Check CustomBob + bob = nodes["CustomBob"] + self.assertEqual(bob["id"], "CustomBob") + self.assertEqual(bob["displayName"], "Custom Bob") + self.assertEqual(bob["source"]["type"], "custom") + self.assertEqual(bob["source"]["data"], "custom://components.py/CustomBob") + self.assertIn("icon", bob) + + # Check groups + groups = data["groups"] + self.assertEqual(len(groups), 2) + + # Find groups by title + custom_group = next((g for g in groups if g["title"] == "Your PyFlyde Nodes"), None) + stdlib_group = next((g for g in groups if g["title"] == "PyFlyde Standard Nodes"), None) + + self.assertIsNotNone(custom_group) + self.assertIsNotNone(stdlib_group) + if custom_group is None or stdlib_group is None: + return + self.assertCountEqual(custom_group["nodeIds"], ["CustomBob", "CustomAlice"]) + self.assertCountEqual(stdlib_group["nodeIds"], list_nodes()) + + def test_gen_json_with_flyde_files(self): + # Create test files with both .py components and copy an existing .flyde file + components_py = ''' +from flyde.io import Input, Output +from flyde.node import Component + +class CustomNode(Component): + """A custom Python node""" + inputs = {"value": Input(description="Input value")} + outputs = {"result": Output(description="Result value")} +''' + + with open(os.path.join(self.temp_dir, "components.py"), "w") as f: + f.write(components_py) + + # Copy an existing .flyde file + test_flyde_source = os.path.join(os.path.dirname(__file__), "..", "tests", "Repeat3Times.flyde") + test_flyde_dest = os.path.join(self.temp_dir, "TestFlow.flyde") + shutil.copy(test_flyde_source, test_flyde_dest) + + # Generate JSON + gen_json(self.temp_dir) + + # Check that the file was created + output_file = os.path.join(self.temp_dir, "flyde-nodes.json") + self.assertTrue(os.path.exists(output_file)) + + # Load and verify the JSON content + with open(output_file, "r") as f: + data = json.load(f) + + # Check structure + self.assertIn("nodes", data) + self.assertIn("groups", data) + + # Check nodes - should have both Python and .flyde nodes + nodes = data["nodes"] + expected_custom_nodes = set(["CustomNode", "TestFlow"] + list_nodes()) + self.assertEqual(set(nodes.keys()), expected_custom_nodes) + + # Check CustomNode (Python) + custom_node = nodes["CustomNode"] + self.assertEqual(custom_node["type"], "code") + self.assertEqual(custom_node["source"]["type"], "custom") + + # Check TestFlow (.flyde) - uses filename as ID + test_flow = nodes["TestFlow"] + self.assertEqual(test_flow["id"], "TestFlow") + self.assertEqual(test_flow["type"], "visual") + self.assertEqual(test_flow["displayName"], "Test Flow") + self.assertEqual(test_flow["icon"], "fa-diagram-project") + self.assertEqual(test_flow["source"]["type"], "file") + self.assertEqual(test_flow["source"]["data"], "TestFlow.flyde") + + # Check groups + groups = data["groups"] + custom_group = next((g for g in groups if g["title"] == "Your PyFlyde Nodes"), None) + self.assertIsNotNone(custom_group) + if custom_group is not None: + self.assertIn("CustomNode", custom_group["nodeIds"]) + self.assertIn("TestFlow", custom_group["nodeIds"]) + + def test_gen_json_only_flyde_files(self): + # Test with only .flyde files (no Python components) using existing files + test_flyde_source_1 = os.path.join(os.path.dirname(__file__), "..", "tests", "Repeat3Times.flyde") + test_flyde_source_2 = os.path.join(os.path.dirname(__file__), "..", "tests", "TestNestedFlow.flyde") + + shutil.copy(test_flyde_source_1, os.path.join(self.temp_dir, "Flow1.flyde")) + shutil.copy(test_flyde_source_2, os.path.join(self.temp_dir, "Flow2.flyde")) + + gen_json(self.temp_dir) + + output_file = os.path.join(self.temp_dir, "flyde-nodes.json") + with open(output_file, "r") as f: + data = json.load(f) + + # Should have .flyde nodes and stdlib nodes + nodes = data["nodes"] + expected_nodes = set(["Flow1", "Flow2"] + list_nodes()) + self.assertEqual(set(nodes.keys()), expected_nodes) + + # Check that .flyde nodes are in the custom group + groups = data["groups"] + custom_group = next((g for g in groups if g["title"] == "Your PyFlyde Nodes"), None) + self.assertIsNotNone(custom_group) + if custom_group is not None: + self.assertIn("Flow1", custom_group["nodeIds"]) + self.assertIn("Flow2", custom_group["nodeIds"]) + + def test_gen_json_empty_directory(self): + # Test with directory containing no components + empty_py = """ +# This file has no Component subclasses +def some_function(): + pass + +class NotAComponent: + pass +""" + + with open(os.path.join(self.temp_dir, "empty.py"), "w") as f: + f.write(empty_py) + + # Generate JSON + gen_json(self.temp_dir) + + # Should create the JSON file with only stdlib nodes if any exist + output_file = os.path.join(self.temp_dir, "flyde-nodes.json") + self.assertTrue(os.path.exists(output_file)) + + with open(output_file, "r") as f: + data = json.load(f) + self.assertIn("nodes", data) + self.assertIn("groups", data) + # At least one stdlib node should be present if stdlib is available + stdlib_group = next((g for g in data["groups"] if g["title"] == "PyFlyde Standard Nodes"), None) + self.assertIsNotNone(stdlib_group) + if stdlib_group is None: + return + self.assertGreater(len(stdlib_group["nodeIds"]), 0) + + def test_gen_json_only_custom_nodes(self): + # Test with only custom nodes (no stdlib nodes in this directory) + components_py = ''' +from flyde.io import Input, Output +from flyde.node import Component + +class CustomNode1(Component): + """First custom node""" + inputs = {"inp": Input(description="Input")} + outputs = {"out": Output(description="Output")} + +class CustomNode2(Component): + """Second custom node""" + outputs = {"result": Output(description="Result")} +''' + + with open(os.path.join(self.temp_dir, "components.py"), "w") as f: + f.write(components_py) + + gen_json(self.temp_dir) + + output_file = os.path.join(self.temp_dir, "flyde-nodes.json") + with open(output_file, "r") as f: + data = json.load(f) + + # Should have at least the custom nodes group + groups = data["groups"] + custom_group = next((g for g in groups if g["title"] == "Your PyFlyde Nodes"), None) + self.assertIsNotNone(custom_group) + if custom_group is None: + return + self.assertCountEqual(custom_group["nodeIds"], ["CustomNode1", "CustomNode2"]) + + def test_gen_json_only_stdlib_nodes(self): + # Test with only stdlib nodes + components_py = ''' +from flyde.io import Input, Output +from flyde.node import Component + +class InlineValue(Component): + """Override InlineValue""" + outputs = {"value": Output(description="Value")} + +class Conditional(Component): + """Override Conditional""" + inputs = {"condition": Input(description="Condition")} + outputs = {"result": Output(description="Result")} +''' + + with open(os.path.join(self.temp_dir, "components.py"), "w") as f: + f.write(components_py) + + gen_json(self.temp_dir) + + output_file = os.path.join(self.temp_dir, "flyde-nodes.json") + with open(output_file, "r") as f: + data = json.load(f) + + # Should have stdlib group + groups = data["groups"] + stdlib_group = next((g for g in groups if g["title"] == "PyFlyde Standard Nodes"), None) + self.assertIsNotNone(stdlib_group) + if stdlib_group is None: + return + self.assertCountEqual(stdlib_group["nodeIds"], list_nodes()) diff --git a/tests/test_flow.py b/tests/test_flow.py index a855f65..550cde4 100644 --- a/tests/test_flow.py +++ b/tests/test_flow.py @@ -1,7 +1,8 @@ -from queue import Queue import unittest -from flyde.io import EOF +from queue import Queue + from flyde.flow import Flow +from flyde.io import EOF class TestIsolatedFlow(unittest.TestCase): @@ -103,6 +104,81 @@ def test_with_component(self): flow.stopped.wait() self.assertTrue(flow.stopped.is_set()) + +class TestCustomLoadFlow(unittest.TestCase): + def test_custom_component_loading(self): + """Test loading components using custom:// source format.""" + test_case = { + "inputs": ["hello", "world", EOF], + "outputs": ["HELLO", "WORLD", EOF], + } + flow = Flow.from_file("tests/TestCustomLoad.flyde") + + in_q = flow.node.inputs["input"].queue + out_q = Queue() + flow.node.outputs["output"].connect(out_q) + + flow.run() + + for inp, expected_out in zip(test_case["inputs"], test_case["outputs"]): + in_q.put(inp) + if expected_out == EOF: + actual_out = out_q.get() + self.assertEqual(EOF, actual_out) + else: + actual_out = out_q.get() + self.assertEqual(expected_out, actual_out) + + flow.stopped.wait() + self.assertTrue(flow.stopped.is_set()) + + def test_custom_component_loading_edge_cases(self): + """Test edge cases in custom component loading.""" + from flyde.flow import Flow + from flyde.node import InstanceArgs, InstanceSource, InstanceSourceType + + flow = Flow({}) + + # Test malformed custom path (no class name) + args = InstanceArgs( + id="test", + display_name="Test", + stopped=None, + config={}, + source=InstanceSource( + type=InstanceSourceType.CUSTOM, data="custom://tests/components.py/" + ), + ) + with self.assertRaises(Exception): + flow.create_component("Echo", args) + + # Test invalid module path + args = InstanceArgs( + id="test", + display_name="Test", + stopped=None, + config={}, + source=InstanceSource( + type=InstanceSourceType.CUSTOM, + data="custom://tests/nonexistent.py/Echo", + ), + ) + with self.assertRaises(Exception): + flow.create_component("Echo", args) + + # Test valid custom path + args = InstanceArgs( + id="test", + display_name="Test", + stopped=None, + config={}, + source=InstanceSource( + type=InstanceSourceType.CUSTOM, data="custom://tests/components.py/Echo" + ), + ) + component = flow.create_component("Echo", args) + self.assertIsNotNone(component) + def test_with_graph(self): test_case = { "inputs": ["John", EOF], diff --git a/tests/test_io.py b/tests/test_io.py index bdebb89..61210ff 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -1,12 +1,15 @@ import unittest from queue import Queue + from flyde.io import ( - Input, - InputMode, - Output, EOF, Connection, ConnectionNode, + Input, + InputConfig, + InputMode, + InputType, + Output, Requiredness, ) @@ -275,6 +278,57 @@ def test_ref_count(self): input.dec_ref_count() self.assertEqual(input.ref_count, 0) + def test_apply_config(self): + test_cases = [ + { + "name": "dynamic input config", + "config": InputConfig(type=InputType.DYNAMIC, value=None), + "expected": {"value": None, "input_mode": InputMode.QUEUE, "type": None}, + }, + { + "name": "number input config", + "config": InputConfig(type=InputType.NUMBER, value=42), + "expected": {"value": 42, "input_mode": InputMode.STICKY, "type": int}, + }, + { + "name": "boolean input config", + "config": InputConfig(type=InputType.BOOLEAN, value=True), + "expected": {"value": True, "input_mode": InputMode.STICKY, "type": bool}, + }, + { + "name": "json input config", + "config": InputConfig(type=InputType.JSON, value={"key": "value"}), + "expected": {"value": {"key": "value"}, "input_mode": InputMode.STICKY, "type": dict}, + }, + { + "name": "string input config", + "config": InputConfig(type=InputType.STRING, value="test"), + "expected": {"value": "test", "input_mode": InputMode.STICKY, "type": str}, + }, + { + "name": "input config with preset type", + "config": InputConfig(type=InputType.NUMBER, value=42), + "preset_type": float, + "expected": {"value": 42, "input_mode": InputMode.STICKY, "type": float}, + }, + ] + + for test_case in test_cases: + with self.subTest(case=test_case["name"]): + # Create new input instance for each test + if "preset_type" in test_case: + input_inst = Input(type=test_case["preset_type"]) + else: + input_inst = Input() + + # Apply the config + input_inst.apply_config(test_case["config"]) + + # Check all expected values + self.assertEqual(input_inst._value, test_case["expected"]["value"]) + self.assertEqual(input_inst._input_mode, test_case["expected"]["input_mode"]) + self.assertEqual(input_inst.type, test_case["expected"]["type"]) + class TestOutput(unittest.TestCase): def setUp(self): diff --git a/tests/test_node.py b/tests/test_node.py index 02b967c..0342c7b 100644 --- a/tests/test_node.py +++ b/tests/test_node.py @@ -1,9 +1,10 @@ import threading import unittest -from threading import Thread from queue import Queue -from flyde.io import Input, InputMode, Output, EOF -from flyde.node import Component +from threading import Thread + +from flyde.io import EOF, Input, InputConfig, InputMode, InputType, Output +from flyde.node import Component, InstanceArgs from tests.components import RepeatWordNTimes @@ -116,44 +117,16 @@ def test_static_input(self): self.assertEqual(out_q.get(), EOF) self.assertEqual(in_q.qsize(), 0) - def test_to_ts(self): - self.maxDiff = None - - def expected_typescript(name): - return """export const {NAME}: CodeNode = { - id: "{NAME}", - description: "A component that has both inputs and outputs and a sticky input.", - inputs: { - word: { description: "The input" }, - times: { description: "The number of times to repeat the input" } - }, - outputs: { - out: { description: "The output" } - }, - run: () => { return; }, -}; - -""".replace( - "{NAME}", name - ) - - self.assertEqual( - RepeatWordNTimes.to_ts(), expected_typescript("RepeatWordNTimes") - ) - self.assertEqual( - RepeatWordNTimes.to_ts("RepeatWord"), expected_typescript("RepeatWord") - ) - def test_from_yaml(self): yaml = { "id": "repeat", "nodeId": "RepeatWordNTimes", - "inputConfig": {}, + "config": {}, "displayName": "Repeat", } - def factory(class_name: str, args: dict): - return RepeatWordNTimes(**args) + def factory(class_name: str, args: InstanceArgs): + return RepeatWordNTimes(**args.to_dict()) node = Component.from_yaml(factory, yaml) self.assertEqual(node._id, "repeat") @@ -165,18 +138,17 @@ def test_from_yaml_with_macrodata(self): yaml = { "id": "repeat", "nodeId": "RepeatWordNTimes", - "inputConfig": {}, "displayName": "Repeat", - "macroData": {"value": 100, "key": "foo"}, + "config": {"value": 100, "key": "foo"}, } - def factory(class_name: str, args: dict): - self.assertEqual(args["macro_data"]["value"], yaml["macroData"]["value"]) - self.assertEqual(args["macro_data"]["key"], yaml["macroData"]["key"]) - # Drop macro_data from the args, otherwise there will be an exception - # because the constructor doesn't support it. - del args["macro_data"] - return RepeatWordNTimes(**args) + def factory(class_name: str, args: InstanceArgs): + self.assertIsNotNone(args.config) + if args.config is None: + raise ValueError("macro_data is None") + self.assertEqual(args.config["value"], yaml["config"]["value"]) + self.assertEqual(args.config["key"], yaml["config"]["key"]) + return RepeatWordNTimes(**args.to_dict()) node = Component.from_yaml(factory, yaml) self.assertEqual(node._id, "repeat") @@ -189,11 +161,90 @@ def test_to_dict(self): expected = { "id": "repeat", "nodeId": "RepeatWordNTimes", - "inputConfig": {}, + "config": {}, "displayName": "Repeat", } self.assertEqual(node.to_dict(), expected) + def test_parse_config_with_type_only(self): + config = {"times": {"type": "number"}, "word": {"type": "string", "value": "default"}} + + node = RepeatWordNTimes(id="repeat", display_name="Repeat", config=config) + + self.assertIn("times", node._config) + self.assertIsInstance(node._config["times"], InputConfig) + self.assertEqual(node._config["times"].type, InputType.NUMBER) + self.assertIsNone(node._config["times"].value) + + self.assertIn("word", node._config) + self.assertIsInstance(node._config["word"], InputConfig) + self.assertEqual(node._config["word"].type, InputType.STRING) + self.assertEqual(node._config["word"].value, "default") + + +class AllStickyInputsComponent(Component): + """A component with only sticky inputs to test single execution.""" + + inputs = { + "a": Input(description="First sticky input", type=int, mode=InputMode.STICKY, value=5), + "b": Input(description="Second sticky input", type=int, mode=InputMode.STICKY, value=10), + } + + outputs = {"result": Output(description="Result of operation", type=int)} + + def process(self, a: int, b: int) -> dict[str, int]: + return {"result": a + b} + + +class TestAllStickyInputsComponent(unittest.TestCase): + def test_all_sticky_inputs_run_once(self): + """Test that a component with all sticky inputs runs only once.""" + node = AllStickyInputsComponent(id="sticky_test", display_name="Sticky Test") + + # Connect an output queue to capture results + out_q = Queue() + node.outputs["result"].connect(out_q) + + # Run the component + node.run() + + # Check that it produced a single result and then EOF + self.assertEqual(out_q.get(), 15) # 5 + 10 + self.assertEqual(out_q.get(), EOF) + + # Verify that the component has stopped + node.stopped.wait(timeout=1) + self.assertTrue(node.stopped.is_set(), "Component should have stopped after one execution") + + def test_all_sticky_inputs_with_config(self): + """Test that a component with all sticky inputs initializes from config and runs once.""" + from flyde.io import InputConfig, InputType + + # Create node with config values + node = AllStickyInputsComponent( + id="sticky_test", + display_name="Sticky Test", + config={ + "a": InputConfig(type=InputType.NUMBER, value=20), + "b": InputConfig(type=InputType.NUMBER, value=30), + }, + ) + + # Connect an output queue to capture results + out_q = Queue() + node.outputs["result"].connect(out_q) + + # Run the component + node.run() + + # Check that it produced a single result with the configured values and then EOF + self.assertEqual(out_q.get(), 50) # 20 + 30 + self.assertEqual(out_q.get(), EOF) + + # Verify that the component has stopped + node.stopped.wait(timeout=1) + self.assertTrue(node.stopped.is_set(), "Component should have stopped after one execution") + class SourceComponent(Component): """A component that only has outputs.""" diff --git a/tests/test_stdlib.py b/tests/test_stdlib.py index 670007c..492d549 100644 --- a/tests/test_stdlib.py +++ b/tests/test_stdlib.py @@ -1,9 +1,19 @@ import unittest from queue import Queue from types import SimpleNamespace +from unittest.mock import patch, MagicMock +from urllib import error +from http import client -from flyde.io import EOF -from flyde.stdlib import InlineValue, Conditional, GetAttribute +from flyde.io import EOF, InputConfig, InputType +from flyde.nodes import ( + Conditional, + GetAttribute, + Http, + InlineValue, + _ConditionConfig, + _ConditionType, +) class TestInlineValue(unittest.TestCase): @@ -13,7 +23,10 @@ def test_inline_value(self): "outputs": {"value": "Hello"}, } out_q = Queue() - node = InlineValue(macro_data={"value": "Hello"}, id="test_inline_value") + node = InlineValue( + id="test_inline_value", + config={"value": InputConfig(type=InputType.STRING, value="Hello")}, + ) node.outputs["value"].connect(out_q) node.run() self.assertEqual(test_case["outputs"]["value"], out_q.get()) @@ -27,8 +40,8 @@ def test_inline_value_dict(self): } out_q = Queue() node = InlineValue( - macro_data={"value": {"type": "string", "value": "Hello"}}, id="test_inline_value", + config={"value": InputConfig(type=InputType.STRING, value="Hello")}, ) node.outputs["value"].connect(out_q) node.run() @@ -42,17 +55,14 @@ def test_conditional(self): test_cases = [ { "name": "equal static string", - "yml": { - "leftOperand": { - "type": "static", - "value": "Apple", - }, - "rightOperand": { - "type": "dynamic", - }, - "condition": { - "type": "EQUAL", - }, + "config": { + "leftOperand": InputConfig(type=InputType.STRING, value="Apple"), + "rightOperand": InputConfig( + type=InputType.DYNAMIC, + ), + "condition": _ConditionConfig( + type=_ConditionType.Equal, + ), }, "inputs": { "leftOperand": [], @@ -66,16 +76,16 @@ def test_conditional(self): }, { "name": "not equal dynamic string", - "yml": { - "leftOperand": { - "type": "dynamic", - }, - "rightOperand": { - "type": "dynamic", - }, - "condition": { - "type": "NOT_EQUAL", - }, + "config": { + "leftOperand": InputConfig( + type=InputType.DYNAMIC, + ), + "rightOperand": InputConfig( + type=InputType.DYNAMIC, + ), + "condition": _ConditionConfig( + type=_ConditionType.NotEqual, + ), }, "inputs": { "leftOperand": ["Apple", "Banana", "apple", "Grape", EOF], @@ -88,17 +98,14 @@ def test_conditional(self): }, { "name": "contains static string", - "yml": { - "leftOperand": { - "type": "dynamic", - }, - "rightOperand": { - "type": "static", - "value": "Apple", - }, - "condition": { - "type": "CONTAINS", - }, + "config": { + "leftOperand": InputConfig( + type=InputType.DYNAMIC, + ), + "rightOperand": InputConfig(type=InputType.STRING, value="Apple"), + "condition": _ConditionConfig( + type=_ConditionType.Contains, + ), }, "inputs": { "leftOperand": [ @@ -117,17 +124,14 @@ def test_conditional(self): }, { "name": "not contains static string", - "yml": { - "leftOperand": { - "type": "dynamic", - }, - "rightOperand": { - "type": "static", - "value": "Apple", - }, - "condition": { - "type": "NOT_CONTAINS", - }, + "config": { + "leftOperand": InputConfig( + type=InputType.DYNAMIC, + ), + "rightOperand": InputConfig(type=InputType.STRING, value="Apple"), + "condition": _ConditionConfig( + type=_ConditionType.NotContains, + ), }, "inputs": { "leftOperand": [ @@ -146,17 +150,14 @@ def test_conditional(self): }, { "name": "regex matches static", - "yml": { - "leftOperand": { - "type": "dynamic", - }, - "rightOperand": { - "type": "static", - "value": "^[A-Z]", - }, - "condition": { - "type": "REGEX_MATCHES", - }, + "config": { + "leftOperand": InputConfig( + type=InputType.DYNAMIC, + ), + "rightOperand": InputConfig(type=InputType.STRING, value="^[A-Z]"), + "condition": _ConditionConfig( + type=_ConditionType.RegexMatches, + ), }, "inputs": { "leftOperand": [ @@ -176,17 +177,16 @@ def test_conditional(self): }, { "name": "exists", - "yml": { - "leftOperand": { - "type": "dynamic", - }, - "rightOperand": { - "type": "static", - "value": "this is not important", - }, - "condition": { - "type": "EXISTS", - }, + "config": { + "leftOperand": InputConfig( + type=InputType.DYNAMIC, + ), + "rightOperand": InputConfig( + type=InputType.STRING, value="this is not important" + ), + "condition": _ConditionConfig( + type=_ConditionType.Exists, + ), }, "inputs": { "leftOperand": ["Apple", "", " ", " ", "banana", EOF], @@ -199,17 +199,16 @@ def test_conditional(self): }, { "name": "does not exist", - "yml": { - "leftOperand": { - "type": "dynamic", - }, - "rightOperand": { - "type": "static", - "value": "this is not important", - }, - "condition": { - "type": "NOT_EXISTS", - }, + "config": { + "leftOperand": InputConfig( + type=InputType.DYNAMIC, + ), + "rightOperand": InputConfig( + type=InputType.STRING, value="this is not important" + ), + "condition": _ConditionConfig( + type=_ConditionType.NotExists, + ), }, "inputs": { "leftOperand": ["Apple", "", " ", " ", "banana", EOF], @@ -222,16 +221,16 @@ def test_conditional(self): }, { "name": "unsupported condition type", - "yml": { - "leftOperand": { - "type": "dynamic", - }, - "rightOperand": { - "type": "dynamic", - }, + "config": { + "leftOperand": InputConfig( + type=InputType.DYNAMIC, + ), + "rightOperand": InputConfig( + type=InputType.DYNAMIC, + ), "condition": { - "type": "UNSUPPORTED", - }, + "type": "UNSUPPORTED" + }, # Will cause a ValueError in parse_config }, "inputs": { "leftOperand": ["Apple", "Banana", "apple", EOF], @@ -251,10 +250,12 @@ def test_conditional(self): if "raises" in test_case and test_case["raises"] is not None: with self.assertRaises(test_case["raises"]): - node = Conditional(test_case["yml"], id="test_conditional") + node = Conditional( + id="test_conditional", config=test_case["config"] + ) continue - node = Conditional(test_case["yml"], id="test_conditional") + node = Conditional(id="test_conditional", config=test_case["config"]) left_q = node.inputs["leftOperand"].queue right_q = node.inputs["rightOperand"].queue node.outputs["true"].connect(true_q) @@ -288,9 +289,11 @@ def test_get_attribute(self): test_cases = [ { "name": "static attribute from a dict", - "key": { - "type": "static", - "value": "name", + "config": { + "key": InputConfig( + type=InputType.STRING, + value="name", + ), }, "inputs": { "object": [ @@ -305,9 +308,11 @@ def test_get_attribute(self): }, { "name": "sticky attribute from an object", - "key": { - "type": "sticky", - "value": "name", + "config": { + "key": InputConfig( + type=InputType.STRING, + value="name", + ), }, "inputs": { "object": [ @@ -316,13 +321,18 @@ def test_get_attribute(self): SimpleNamespace(nananan="Charlie"), EOF, ], - "key": ["name"], + "key": ["name", EOF], }, "outputs": ["Alice", "Bob", None, EOF], }, { "name": "dynamic attribute from a dict", - "key": {}, + "config": { + "key": InputConfig( + type=InputType.DYNAMIC, + value=None, + ), + }, "inputs": { "object": [ {"name": "Alice"}, @@ -335,10 +345,12 @@ def test_get_attribute(self): "outputs": ["Alice", "Bob", "Charlie", EOF], }, { - "name": "sticky attribute and non-object", - "key": { - "type": "sticky", - "value": "name", + "name": "static attribute and non-object", + "config": { + "key": InputConfig( + type=InputType.STRING, + value="name", + ), }, "inputs": { "object": [ @@ -347,15 +359,17 @@ def test_get_attribute(self): 123, EOF, ], - "key": ["name"], + "key": ["name", EOF], }, "outputs": ["Alice", None, None, EOF], }, { "name": "nested attribute with dot key notation", - "key": { - "type": "static", - "value": "address.city", + "config": { + "key": InputConfig( + type=InputType.STRING, + value="address.city", + ), }, "inputs": { "object": [ @@ -370,9 +384,11 @@ def test_get_attribute(self): }, { "name": "nested 3 levels deep", - "key": { - "type": "static", - "value": "address.city.zip", + "config": { + "key": InputConfig( + type=InputType.STRING, + value="address.city.zip", + ), }, "inputs": { "object": [ @@ -390,9 +406,9 @@ def test_get_attribute(self): for test_case in test_cases: attr_q = Queue() out_q = Queue() - node = GetAttribute( - macro_data={"key": test_case["key"]}, id="test_get_attribute" - ) + config = test_case["config"] + + node = GetAttribute(id="test_get_attribute", config=config) obj_q = node.inputs["object"].queue if len(test_case["inputs"]["key"]) > 0: attr_q = node.inputs["key"].queue @@ -405,3 +421,260 @@ def test_get_attribute(self): ): attr_q.put(test_case["inputs"]["key"][i]) self.assertEqual(test_case["outputs"][i], out_q.get()) + + node.stopped.wait() + self.assertTrue(node.stopped.is_set()) + + +class TestHttp(unittest.TestCase): + @patch("urllib.request.urlopen") + def test_http_get(self, mock_urlopen): + mock_response = MagicMock() + mock_response.status = 200 + mock_response.headers = {"Content-Type": "application/json"} + mock_response.read.return_value = b'{"message": "Hello, World!"}' + mock_response.__enter__.return_value = mock_response + mock_urlopen.return_value = mock_response + + test_case = { + "name": "simple GET request", + "config": { + "method": InputConfig(type=InputType.STRING, value="GET"), + "url": InputConfig( + type=InputType.STRING, value="https://example.com/api" + ), + "headers": InputConfig( + type=InputType.JSON, value={"User-Agent": "PyFlyde Test"} + ), + "params": InputConfig(type=InputType.JSON, value={"q": "test"}), + }, + } + + data_q = Queue() + + node = Http(id="test_http", config=test_case["config"]) + node.outputs["data"].connect(data_q) + + node.process( + url="https://example.com/api", + method="GET", + headers={"User-Agent": "PyFlyde Test"}, + params={"q": "test"}, + ) + + self.assertEqual({"message": "Hello, World!"}, data_q.get_nowait()) + + # Verify the mock was called with the expected arguments + mock_urlopen.assert_called_once() + args, _ = mock_urlopen.call_args + self.assertTrue("https://example.com/api?q=test" in str(args[0].full_url)) + self.assertEqual("GET", args[0].method) + + # Check that the User-Agent header was set + user_agent = None + for key, value in args[0].headers.items(): + if key.lower() == "user-agent": + user_agent = value + break + self.assertEqual("PyFlyde Test", user_agent) + + @patch("urllib.request.urlopen") + def test_http_html_response(self, mock_urlopen): + mock_response = MagicMock() + mock_response.status = 200 + mock_response.headers = {"Content-Type": "text/html; charset=utf-8"} + mock_response.read.return_value = ( + b"

Test Page

" + ) + mock_response.__enter__.return_value = mock_response + mock_urlopen.return_value = mock_response + + data_q = Queue() + + node = Http( + id="test_http", + config={ + "method": InputConfig(type=InputType.STRING, value="GET"), + "url": InputConfig(type=InputType.STRING, value="https://example.com"), + }, + ) + node.outputs["data"].connect(data_q) + + node.process(url="https://example.com", method="GET") + + self.assertEqual( + "

Test Page

", + data_q.get_nowait(), + ) + + mock_urlopen.assert_called_once() + args, _ = mock_urlopen.call_args + self.assertEqual("https://example.com", str(args[0].full_url)) + self.assertEqual("GET", args[0].method) + + @patch("urllib.request.urlopen") + def test_http_binary_response(self, mock_urlopen): + mock_response = MagicMock() + mock_response.status = 200 + mock_response.headers = {"Content-Type": "application/octet-stream"} + binary_data = ( + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00" # Start of a PNG file + ) + mock_response.read.return_value = binary_data + mock_response.__enter__.return_value = mock_response + mock_urlopen.return_value = mock_response + + data_q = Queue() + + node = Http( + id="test_http", + config={ + "method": InputConfig(type=InputType.STRING, value="GET"), + "url": InputConfig( + type=InputType.STRING, value="https://example.com/image.png" + ), + }, + ) + node.outputs["data"].connect(data_q) + + node.process(url="https://example.com/image.png", method="GET") + + # Binary data should be returned as is + self.assertEqual(binary_data, data_q.get_nowait()) + + mock_urlopen.assert_called_once() + args, _ = mock_urlopen.call_args + self.assertEqual("https://example.com/image.png", str(args[0].full_url)) + self.assertEqual("GET", args[0].method) + + @patch("urllib.request.urlopen") + def test_http_non_utf8_encoding(self, mock_urlopen): + mock_response = MagicMock() + mock_response.status = 200 + mock_response.headers = {"Content-Type": "text/html; charset=ISO-8859-1"} + # Latin-1 encoded text with special characters + latin1_data = b"Espa\xf1ol Fran\xe7ais Portugu\xeas" + mock_response.read.return_value = latin1_data + mock_response.__enter__.return_value = mock_response + mock_urlopen.return_value = mock_response + + data_q = Queue() + + node = Http( + id="test_http", + config={ + "method": InputConfig(type=InputType.STRING, value="GET"), + "url": InputConfig( + type=InputType.STRING, value="https://example.com/latin1.html" + ), + }, + ) + node.outputs["data"].connect(data_q) + + node.process(url="https://example.com/latin1.html", method="GET") + + # Should be properly decoded using ISO-8859-1 charset + self.assertEqual("Español Français Português", data_q.get_nowait()) + + mock_urlopen.assert_called_once() + args, _ = mock_urlopen.call_args + self.assertEqual("https://example.com/latin1.html", str(args[0].full_url)) + self.assertEqual("GET", args[0].method) + + @patch("urllib.request.urlopen") + def test_http_post(self, mock_urlopen): + mock_response = MagicMock() + mock_response.status = 201 + mock_response.headers = {"Content-Type": "application/json"} + mock_response.read.return_value = b'{"id": 1, "success": true}' + mock_response.__enter__.return_value = mock_response + mock_urlopen.return_value = mock_response + + test_case = { + "name": "POST request with data", + "config": { + "method": InputConfig(type=InputType.STRING, value="POST"), + "url": InputConfig( + type=InputType.STRING, value="https://example.com/api/users" + ), + "headers": InputConfig( + type=InputType.JSON, value={"User-Agent": "PyFlyde Test"} + ), + "data": InputConfig( + type=InputType.JSON, + value={"name": "Test User", "email": "test@example.com"}, + ), + }, + } + + data_q = Queue() + + node = Http(id="test_http", config=test_case["config"]) + node.outputs["data"].connect(data_q) + + node.process( + url="https://example.com/api/users", + method="POST", + headers={"User-Agent": "PyFlyde Test"}, + data={"name": "Test User", "email": "test@example.com"}, + ) + + self.assertEqual({"id": 1, "success": True}, data_q.get_nowait()) + + # Verify the mock was called with the expected arguments + mock_urlopen.assert_called_once() + args, kwargs = mock_urlopen.call_args + self.assertEqual("https://example.com/api/users", str(args[0].full_url)) + self.assertEqual("POST", args[0].method) + + # Check that the User-Agent header was set + user_agent = None + for key, value in args[0].headers.items(): + if key.lower() == "user-agent": + user_agent = value + break + self.assertEqual("PyFlyde Test", user_agent) + + self.assertEqual( + b'{"name": "Test User", "email": "test@example.com"}', kwargs["data"] + ) + + @patch("urllib.request.urlopen") + def test_http_error(self, mock_urlopen): + # Create a mock headers object for HTTPError + headers = client.HTTPMessage() + headers.add_header("Content-Type", "text/plain") + + mock_urlopen.side_effect = error.HTTPError( + url="https://example.com/api/error", + code=404, + msg="Not Found", + hdrs=headers, + fp=None, + ) + + test_case = { + "name": "HTTP error handling", + "config": { + "method": InputConfig(type=InputType.STRING, value="GET"), + "url": InputConfig( + type=InputType.STRING, value="https://example.com/api/error" + ), + }, + } + + data_q = Queue() + + node = Http(id="test_http", config=test_case["config"]) + node.outputs["data"].connect(data_q) + + with self.assertRaises(error.HTTPError) as context: + node.process(url="https://example.com/api/error", method="GET") + + self.assertEqual(404, context.exception.code) + self.assertEqual("Not Found", context.exception.msg) + + # Verify the mock was called + mock_urlopen.assert_called_once() + args, _ = mock_urlopen.call_args + self.assertEqual("https://example.com/api/error", str(args[0].full_url))