EvalProvider using RestrictedPython [Depends on #519] (#520)

* Fresh diff

* remove instructionhandler

* updated internal interface to make all tests pass

* fixed caching tests

* updated llm.ipynb

* removed unnecessarily defensive validation

* updated tool call decoding to use concrete type of tool result instead of annotations

* updated completions to fix basic type errors

* updated call assistant to handle decoding tool calls

* dropped stale comments

* moved model and param model back to internals of `completions`

* added default encodable instance for Callable

* fixed type errors

* update to use more structured type for synthesis

* updated callable encoding tests

* s/TypeError/NotImplementedError

* simplified smart constructor

* bare callables not allowed

* droped synthesis and removed encoding_instructions

* fixed imports

* fixed imports and tests

* added restricted python again

* added test for custom policies for restricted python

* more specific arguments to RestrictedEvalProvider and made exec more customisable

* reverted flags for customizing rglobals, using same rglobals for local and globals for exec, fixing bug

* fixed failing tests (env was not being mutated)

* updated restricted python to be a llm dependency

---------

Co-authored-by: Eli <eli@basis.ai>

Author: kiranandcode

effectful/handlers/llm/evaluation.py

@@ -5,6 +5,14 @@
 from types import CodeType
 from typing import Any
 
+from RestrictedPython import (
+    Eval,
+    Guards,
+    RestrictingNodeTransformer,
+    compile_restricted,
+    safe_globals,
+)
+
 from effectful.ops.syntax import ObjectInterpretation, defop, implements
 
 
@@ -86,3 +94,80 @@ def exec(
 
         # Execute module-style so top-level defs land in `env`.
         builtins.exec(bytecode, env, env)
+
+
+class RestrictedEvalProvider(ObjectInterpretation):
+    """
+    Safer provider using RestrictedPython.
+
+    RestrictedPython is not a complete sandbox, but it enforces a restricted
+    language subset and expects you to provide a constrained exec environment.
+
+    policy : dict[str, Any], optional
+        RestrictedPython compile_restricted policy for compilation
+    """
+
+    policy: type[RestrictingNodeTransformer] | None = None
+
+    def __init__(
+        self,
+        *,
+        policy: type[RestrictingNodeTransformer] | None = None,
+    ):
+        self.policy = policy
+
+    @implements(parse)
+    def parse(self, source: str, filename: str) -> ast.Module:
+        # Keep inspect.getsource() working for dynamically-defined objects.
+        linecache.cache[filename] = (
+            len(source),
+            None,
+            source.splitlines(True),
+            filename,
+        )
+        return ast.parse(source, filename=filename, mode="exec")
+
+    @implements(compile)
+    def compile(self, module: ast.Module, filename: str) -> CodeType:
+        # RestrictedPython can compile from an AST directly.
+        return compile_restricted(
+            module,
+            filename=filename,
+            mode="exec",
+            policy=self.policy or RestrictingNodeTransformer,
+        )
+
+    @implements(exec)
+    def exec(
+        self,
+        bytecode: CodeType,
+        env: dict[str, Any],
+    ) -> None:
+        # Build restricted globals from RestrictedPython's defaults
+        rglobals: dict[str, Any] = safe_globals.copy()
+
+        # Enable class definitions (required for Python 3)
+        rglobals["__metaclass__"] = type
+        rglobals["__name__"] = "restricted"
+
+        # Layer `env` on top (without letting callers replace the restricted builtins).
+        rglobals.update({k: v for k, v in env.items() if k != "__builtins__"})
+
+        # Enable for loops and comprehensions
+        rglobals["_getiter_"] = Eval.default_guarded_getiter
+        # Enable sequence unpacking in comprehensions and for loops
+        rglobals["_iter_unpack_sequence_"] = Guards.guarded_iter_unpack_sequence
+
+        rglobals["getattr"] = Guards.safer_getattr
+        rglobals["setattr"] = Guards.guarded_setattr
+        rglobals["_write_"] = lambda x: x
+
+        # Track keys before execution to identify new definitions
+        keys_before = set(rglobals.keys())
+
+        builtins.exec(bytecode, rglobals, rglobals)
+
+        # Copy newly defined items back to env so caller can access them
+        for key in rglobals:
+            if key not in keys_before:
+                env[key] = rglobals[key]

pyproject.toml

@@ -44,6 +44,7 @@ llm = [
   "litellm",
   "pillow",
   "pydantic",
+  "restrictedpython>=8.1"
 ]
 prettyprinter = ["prettyprinter"]
 docs = [