Add shared state API for cross-worker data sharing

Python workers have isolated namespaces, making it difficult to share
data between calls. This adds an ETS-backed shared state API accessible
from both Python and Erlang.

Python API:
  from erlang import state_set, state_get, state_delete, state_keys
  from erlang import state_incr, state_decr

  state_set('key', {'data': [1, 2, 3]})
  value = state_get('key')
  state_incr('counter')  # atomic increment

Erlang API:
  py:state_store(Key, Value).
  {ok, Value} = py:state_fetch(Key).
  py:state_incr(<<"counter">>).

New files:
- src/py_state.erl - ETS-backed state storage with atomic counters
- examples/shared_state_example.erl - Usage demonstration

Updated docs:
- README.md, getting-started.md, scalability.md, ai-integration.md

Author: benoitc

README.md

@@ -155,6 +155,54 @@ result = erlang.call('my_func', 10, 20)
 All three methods are equivalent. The import and attribute syntaxes provide
 a more natural Python experience.
 
+## Shared State Between Workers
+
+Python workers don't share namespace state, but you can share data via the
+built-in state API:
+
+### From Python
+
+```python
+from erlang import state_set, state_get, state_delete, state_keys
+from erlang import state_incr, state_decr
+
+# Store data (survives across calls, shared between workers)
+state_set('my_key', {'data': [1, 2, 3], 'count': 42})
+
+# Retrieve data
+value = state_get('my_key')  # {'data': [1, 2, 3], 'count': 42}
+
+# Atomic counters (thread-safe, great for metrics)
+state_incr('requests')       # returns 1
+state_incr('requests', 10)   # returns 11
+state_decr('requests')       # returns 10
+
+# List keys
+keys = state_keys()  # ['my_key', 'requests', ...]
+
+# Delete
+state_delete('my_key')
+```
+
+### From Erlang/Elixir
+
+```erlang
+%% Store and fetch
+py:state_store(<<"my_key">>, #{value => 42}).
+{ok, #{value := 42}} = py:state_fetch(<<"my_key">>).
+
+%% Atomic counters
+1 = py:state_incr(<<"hits">>).
+11 = py:state_incr(<<"hits">>, 10).
+10 = py:state_decr(<<"hits">>).
+
+%% List keys and clear
+Keys = py:state_keys().
+py:state_clear().
+```
+
+This is backed by ETS with `{write_concurrency, true}`, so counters are atomic and fast.
+
 ## Async/Await Support
 
 Call Python async functions without blocking:

docs/ai-integration.md

@@ -42,13 +42,17 @@ Embeddings convert text into numerical vectors, enabling semantic search, cluste
 ### Using sentence-transformers
 
 ```erlang
-%% Load a model (do this once at startup)
+%% Load a model - NOTE: This loads in one worker only.
+%% Each worker will lazy-load the model on first use.
 init_embedding_model() ->
     py:exec(<<"
 from sentence_transformers import SentenceTransformer
 model = SentenceTransformer('all-MiniLM-L6-v2')
 ">>).
 
+%% Better pattern: use a module that lazy-loads
+%% and cache embeddings in shared state
+
 %% Generate embedding for a single text
 embed(Text) ->
     {ok, Embedding} = py:eval(
@@ -575,7 +579,44 @@ model = SentenceTransformer('all-MiniLM-L6-v2', device=device)
 ">>).
 ```
 
-### 4. Monitor Rate Limits
+### 4. Cache Embeddings in Shared State
+
+Avoid recomputing embeddings for the same text:
+
+```erlang
+%% Check cache before computing
+embed_cached(Text) ->
+    Key = <<"emb:", (crypto:hash(md5, Text))/binary>>,
+    case py:state_fetch(Key) of
+        {ok, Embedding} ->
+            {ok, Embedding};
+        {error, not_found} ->
+            {ok, Embedding} = py:eval(
+                <<"model.encode(text).tolist()">>,
+                #{text => Text}
+            ),
+            py:state_store(Key, Embedding),
+            {ok, Embedding}
+    end.
+```
+
+Or from Python:
+
+```python
+from erlang import state_get, state_set
+import hashlib
+
+def embed_cached(text):
+    key = f"emb:{hashlib.md5(text.encode()).hexdigest()}"
+    cached = state_get(key)
+    if cached is not None:
+        return cached
+    embedding = model.encode(text).tolist()
+    state_set(key, embedding)
+    return embedding
+```
+
+### 5. Monitor Rate Limits
 
 ```erlang
 %% Check current load before heavy operations

docs/getting-started.md

@@ -65,7 +65,8 @@ def roll_dice(sides=6):
 ```
 
 Note: Definitions made with `exec` are local to the worker that executes them.
-Subsequent calls may go to different workers.
+Subsequent calls may go to different workers. Use [Shared State](#shared-state) to
+share data between workers.
 
 ## Working with Timeouts
 
@@ -105,6 +106,57 @@ Python generators can be streamed efficiently:
 {ok, Chunks} = py:stream(mymodule, generate_data, [arg1, arg2]).
 ```
 
+## Shared State
+
+Python workers don't share namespace state, but you can share data via the
+built-in state API:
+
+```erlang
+%% Store from Erlang
+py:state_store(<<"config">>, #{api_key => <<"secret">>, timeout => 5000}).
+
+%% Read from Python
+ok = py:exec(<<"
+from erlang import state_get
+config = state_get('config')
+print(config['api_key'])
+">>).
+```
+
+### From Python
+
+```python
+from erlang import state_set, state_get, state_delete, state_keys
+from erlang import state_incr, state_decr
+
+# Key-value storage
+state_set('my_key', {'data': [1, 2, 3]})
+value = state_get('my_key')
+
+# Atomic counters (thread-safe)
+state_incr('requests')       # +1, returns new value
+state_incr('requests', 10)   # +10
+state_decr('requests')       # -1
+
+# Management
+keys = state_keys()
+state_delete('my_key')
+```
+
+### From Erlang
+
+```erlang
+py:state_store(Key, Value).
+{ok, Value} = py:state_fetch(Key).
+py:state_remove(Key).
+Keys = py:state_keys().
+
+%% Atomic counters
+1 = py:state_incr(<<"hits">>).
+11 = py:state_incr(<<"hits">>, 10).
+10 = py:state_decr(<<"hits">>).
+```
+
 ## Type Conversions
 
 Values are automatically converted between Erlang and Python:

docs/scalability.md

@@ -32,6 +32,8 @@ When running on a free-threaded Python build (compiled with `--disable-gil`), er
 
 Uses Python's sub-interpreter feature with per-interpreter GIL. Each sub-interpreter has its own GIL, allowing true parallel execution across interpreters.
 
+**Note:** Each sub-interpreter has isolated state. Use the [Shared State](#shared-state) API to share data between workers.
+
 ### Multi-Executor Mode (Python < 3.12)
 
 Runs N executor threads that share the GIL. Requests are distributed round-robin across executors. Good for I/O-bound workloads where Python releases the GIL during I/O operations.
@@ -200,6 +202,34 @@ io:format("Mode: ~p, Executors: ~p~n", [Mode, Executors]).
 io:format("GC stats: ~p~n", [maps:get(gc_stats, Stats)]).
 ```
 
+## Shared State
+
+Since workers (and sub-interpreters) have isolated namespaces, erlang_python provides
+ETS-backed shared state accessible from both Python and Erlang:
+
+```python
+from erlang import state_set, state_get, state_incr, state_decr
+
+# Share configuration across workers
+config = state_get('app_config')
+
+# Thread-safe metrics
+state_incr('requests_total')
+state_incr('bytes_processed', len(data))
+```
+
+```erlang
+%% Set config that all workers can read
+py:state_store(<<"app_config">>, #{model => <<"gpt-4">>, timeout => 30000}).
+
+%% Read metrics
+{ok, Total} = py:state_fetch(<<"requests_total">>).
+```
+
+The state is backed by ETS with `{write_concurrency, true}`, making atomic
+counter operations fast and lock-free. See [Getting Started](getting-started.md#shared-state)
+for the full API.
+
 ## See Also
 
 - [Getting Started](getting-started.md) - Basic usage

examples/shared_state_example.erl

@@ -0,0 +1,109 @@
+#!/usr/bin/env escript
+%%% @doc Shared state example - demonstrates sharing data between Python workers.
+%%%
+%%% Since each Python worker has its own namespace, functions and variables
+%%% defined in one call aren't visible in another. The shared state API
+%%% provides ETS-backed storage accessible from both Python and Erlang.
+%%%
+%%% Prerequisites: rebar3 compile
+%%% Run from project root: escript examples/shared_state_example.erl
+
+-mode(compile).
+
+main(_) ->
+    %% Add the compiled beam files to the code path
+    ScriptDir = filename:dirname(escript:script_name()),
+    ProjectRoot = filename:dirname(ScriptDir),
+    EbinDir = filename:join([ProjectRoot, "_build", "default", "lib", "erlang_python", "ebin"]),
+    true = code:add_pathz(EbinDir),
+
+    {ok, _} = application:ensure_all_started(erlang_python),
+
+    io:format("~n=== Shared State Example ===~n~n"),
+
+    %% Clear any existing state
+    py:state_clear(),
+
+    %% Example 1: Set from Erlang, read from Python
+    io:format("1. Erlang -> Python:~n"),
+    py:state_store(<<"config">>, #{
+        api_key => <<"secret123">>,
+        max_retries => 3,
+        timeout => 5000
+    }),
+    %% Python reads using idiomatic import syntax
+    ok = py:exec(<<"
+from erlang import state_get
+
+config = state_get('config')
+assert config['api_key'] == 'secret123'
+assert config['max_retries'] == 3
+">>),
+    io:format("   Python read config successfully~n"),
+
+    %% Example 2: Set from Python, read from Erlang
+    io:format("~n2. Python -> Erlang:~n"),
+    ok = py:exec(<<"
+from erlang import state_set
+
+state_set('computation_result', {
+    'status': 'complete',
+    'values': [1, 2, 3, 4, 5],
+    'metadata': {'source': 'python', 'timestamp': 1234567890}
+})
+">>),
+    {ok, Result} = py:state_fetch(<<"computation_result">>),
+    io:format("   Result in Erlang: ~p~n", [Result]),
+
+    %% Example 3: Atomic counters - thread-safe increment/decrement
+    io:format("~n3. Atomic counters (thread-safe):~n"),
+
+    %% From Erlang
+    Val1 = py:state_incr(<<"hits">>),
+    io:format("   Erlang incr: ~p~n", [Val1]),
+    Val2 = py:state_incr(<<"hits">>, 10),
+    io:format("   Erlang incr by 10: ~p~n", [Val2]),
+
+    %% From Python
+    ok = py:exec(<<"
+from erlang import state_incr, state_decr
+
+val = state_incr('hits', 5)
+print(f'   Python incr by 5: {val}')
+
+val = state_decr('hits', 3)
+print(f'   Python decr by 3: {val}')
+">>),
+
+    {ok, FinalCount} = py:state_fetch(<<"hits">>),
+    io:format("   Final counter value: ~p~n", [FinalCount]),
+
+    %% Example 4: List all keys from Python
+    io:format("~n4. List keys from Python:~n"),
+    ok = py:exec(<<"
+from erlang import state_keys
+
+keys = state_keys()
+assert 'hits' in keys
+assert 'config' in keys
+">>),
+    Keys = py:state_keys(),
+    io:format("   Keys: ~p~n", [Keys]),
+
+    %% Example 5: Delete from Python
+    io:format("~n5. Delete from Python:~n"),
+    ok = py:exec(<<"
+from erlang import state_delete
+
+state_delete('computation_result')
+">>),
+    case py:state_fetch(<<"computation_result">>) of
+        {error, not_found} -> io:format("   Key deleted successfully~n");
+        {ok, _} -> io:format("   ERROR: Key still exists~n")
+    end,
+
+    %% Cleanup
+    py:state_clear(),
+
+    io:format("~n=== Done ===~n~n"),
+    ok = application:stop(erlang_python).

src/erlang_python_sup.erl

@@ -17,6 +17,7 @@
 %%% Manages the worker pools for Python execution:
 %%% <ul>
 %%%   <li>py_callback - Callback registry for Python to Erlang calls</li>
+%%%   <li>py_state - Shared state storage accessible from Python</li>
 %%%   <li>py_pool - Main worker pool for synchronous Python calls</li>
 %%%   <li>py_async_pool - Worker pool for asyncio coroutines</li>
 %%%   <li>py_subinterp_pool - Worker pool for sub-interpreter parallelism</li>
@@ -42,6 +43,12 @@ init([]) ->
     %% Initialize callback registry ETS table (owned by supervisor for resilience)
     ok = py_callback:init_tab(),
 
+    %% Initialize shared state ETS table (owned by supervisor for resilience)
+    ok = py_state:init_tab(),
+
+    %% Register state functions as callbacks for Python access
+    ok = py_state:register_callbacks(),
+
     %% Callback registry - must start before pool
     CallbackSpec = #{
         id => py_callback,