W-20884161: Add native DataWeave shared library with FFI bindings

refactor python lib

Author: svacas-sfdc

.gitignore

@@ -19,4 +19,6 @@ out/
 .DS_Store
 
 # GraalVM
-.graalvm
+.graalvm
+
+grimoires/

build.gradle

@@ -1,3 +1,13 @@
+buildscript {
+    repositories {
+        gradlePluginPortal()
+        mavenCentral()
+    }
+    dependencies {
+        classpath "org.graalvm.buildtools.native:org.graalvm.buildtools.native.gradle.plugin:0.11.2"
+    }
+}
+
 plugins {
     id "scala"
     id "maven-publish"
@@ -11,6 +21,8 @@ subprojects {
     apply plugin: 'maven-publish'
     apply plugin: 'scala'
 
+    apply plugin: 'org.graalvm.buildtools.native'
+
     group = 'org.mule.weave.native'
     version = nativeVersion
 

gradle.properties

@@ -1,10 +1,10 @@
-weaveVersion=2.11.0-20251023
-weaveTestSuiteVersion=2.11.0-20251023
+weaveVersion=2.12.0-SNAPSHOT
+weaveTestSuiteVersion=2.12.0-SNAPSHOT
 nativeVersion=100.100.100
 scalaVersion=2.12.18
-ioVersion=2.11.0-SNAPSHOT
+ioVersion=2.12.0-SNAPSHOT
 graalvmVersion=24.0.2
-weaveSuiteVersion=2.11.0-20251023
+weaveSuiteVersion=2.12.0-SNAPSHOT
 #Libaries
 scalaTestVersion=3.2.15
 scalaTestPluginVersion=0.33

native-cli/build.gradle

@@ -2,8 +2,6 @@ plugins {
 
     id "com.github.maiflai.scalatest" version "${scalaTestPluginVersion}"
     id 'application'
-    // Apply GraalVM Native Image plugin
-    id 'org.graalvm.buildtools.native' version '0.11.2'
 }
 
 sourceSets {

native-lib/.gitignore

@@ -0,0 +1,2 @@
+python/src/dataweave/native/
+python/dist/

native-lib/README.md

@@ -0,0 +1,222 @@
+# native-lib
+
+## Overview
+
+`native-lib` builds a **GraalVM native shared library** that embeds the MuleSoft **DataWeave runtime** and exposes a small C-compatible API.
+
+The main purpose is to allow non-JVM consumers (most notably the Python package in `native-lib/python`) to execute DataWeave scripts **without running a JVM**, while still using the official DataWeave runtime.
+
+## Architecture (GraalVM + FFI)
+
+```
+┌─────────────────────────────────────────────┐
+│              Python Process                 │
+│                                             │
+│  ┌────────────────────────────────────────┐ │
+│  │  Application Script                    │ │
+│  │  - Python: ctypes                      │ │
+│  └──────────────┬─────────────────────────┘ │
+│                 │                           │
+│                 │ FFI Call                  │
+│                 ▼                           │
+│  ┌────────────────────────────────────────┐ │
+│  │  Native Shared Library (dwlib)         │ │
+│  │  ┌──────────────────────────────────┐  │ │
+│  │  │  GraalVM Isolate                 │  │ │
+│  │  │  - NativeLib.run_script()        │  │ │
+│  │  │  - DataWeave script execution    │  │ │
+│  │  └──────────────────────────────────┘  │ │
+│  └────────────────────────────────────────┘ │
+└─────────────────────────────────────────────┘
+```
+
+## Building with Gradle
+
+### Prerequisites
+
+- A GraalVM distribution installed that includes `native-image`.
+- Enough memory for native-image (this build config uses `-J-Xmx6G`).
+
+### Build the shared library
+
+From the repository root:
+
+```bash
+./gradlew :native-lib:nativeCompile
+```
+
+The shared library is produced under:
+
+- `native-lib/build/native/nativeCompile/`
+
+and is named:
+
+- macOS: `dwlib.dylib`
+- Linux: `dwlib.so`
+- Windows: `dwlib.dll`
+
+### Stage the library into the Python package (dev workflow)
+
+```bash
+./gradlew :native-lib:stagePythonNativeLib
+```
+
+This copies `dwlib.*` into:
+
+- `native-lib/python/src/dataweave/native/`
+
+### Build a Python wheel (bundles the native library)
+
+```bash
+./gradlew :native-lib:buildPythonWheel
+```
+
+The wheel will be created in:
+
+- `native-lib/python/dist/`
+
+## Installing for use in a Python project
+
+### Option A: Install the produced wheel (recommended)
+
+After `:native-lib:buildPythonWheel`:
+
+```bash
+python3 -m pip install native-lib/python/dist/dataweave_native-0.0.1-*.whl
+```
+
+This wheel includes the `dwlib.*` shared library inside the Python package.
+
+### Option B: Editable install for development
+
+1. Stage the native library:
+
+```bash
+./gradlew :native-lib:stagePythonNativeLib
+```
+
+2. Install the Python package in editable mode:
+
+```bash
+python3 -m pip install -e native-lib/python
+```
+
+### Option C: Use an externally-built library via an environment variable
+
+If you want to point Python at a specific built artifact, set:
+
+- `DATAWEAVE_NATIVE_LIB=/absolute/path/to/dwlib.(dylib|so|dll)`
+
+The Python module will also try a few fallbacks (including the wheel-bundled location).
+
+## Using the library (Python examples)
+
+All examples below assume:
+
+```python
+import dataweave
+```
+
+### 1) Simple script
+
+```python
+result = dataweave.run_script("2 + 2")
+assert result.success is True
+print(result.get_string())  # "4"
+```
+
+### 2) Script with inputs (no explicit `mimeType`)
+
+Inputs can be plain Python values. The wrapper auto-encodes them as JSON or text.
+
+```python
+result = dataweave.run_script(
+    "num1 + num2",
+    {"num1": 25, "num2": 17},
+)
+print(result.get_string())  # "42"
+```
+
+### 3) Script with inputs (explicit `mimeType`, `charset`, `properties`)
+
+Use an explicit input dict when you need full control over how DataWeave interprets bytes.
+
+```python
+script = "payload.person"
+xml_bytes = b"<?xml version=\"1.0\" encoding=\"UTF-16\"?><person><name>Billy</name><age>31</age></person>".decode("utf-8").encode("utf-16")
+
+result = dataweave.run_script(
+    script,
+    {
+        "payload": {
+            "content": xml_bytes,
+            "mimeType": "application/xml",
+            "charset": "UTF-16",
+            "properties": {
+                "nullValueOn": "empty",
+                "maxAttributeSize": 256
+            },
+        }
+    },
+)
+
+if result.success:
+    print(result.get_string())
+else:
+    print(result.error)
+```
+
+You can also use `InputValue` for the same purpose:
+
+```python
+input_value = dataweave.InputValue(
+    content="1234567",
+    mimeType="application/csv",
+    properties={"header": False, "separator": "4"},
+)
+
+result = dataweave.run_script("in0.column_1[0]", {"in0": input_value})
+print(result.get_string())  # '"567"'
+```
+
+### 4) Reusing a DataWeave context to run multiple scripts quicker
+
+Creating an isolate/runtime has overhead. For repeated executions, reuse a single `DataWeave` instance:
+
+```python
+with dataweave.DataWeave() as dw:
+    r1 = dw.run("2 + 2")
+    r2 = dw.run("x + y", {"x": 10, "y": 32})
+
+    print(r1.get_string())  # "4"
+    print(r2.get_string())  # "42"
+```
+
+### 5) Error handling
+
+There are two common classes of errors:
+
+- The native library cannot be located/loaded.
+- Script compilation/execution fails (reported as an unsuccessful `ExecutionResult`).
+
+```python
+try:
+    result = dataweave.run_script("invalid syntax here")
+
+    if not result.success:
+        raise dataweave.DataWeaveError(result.error or "Unknown DataWeave error")
+
+    print(result.get_string())
+
+except dataweave.DataWeaveLibraryNotFoundError as e:
+    # Build it (and/or install a wheel) first.
+    # Example build command (from repo root): ./gradlew :native-lib:nativeCompile
+    raise
+
+except dataweave.DataWeaveError:
+    raise
+
+finally:
+    # Optional: if you used the global API and want to force cleanup
+    dataweave.cleanup()
+```

native-lib/build.gradle

@@ -0,0 +1,97 @@
+dependencies {
+  api group: 'org.mule.weave', name: 'runtime', version: weaveVersion
+  api group: 'org.mule.weave', name: 'core-modules', version: weaveVersion
+
+  implementation group: 'org.mule.weave', name: 'parser', version: weaveVersion
+  implementation group: 'org.mule.weave', name: 'wlang', version: weaveVersion
+  compileOnly group: 'org.graalvm.sdk', name: 'graal-sdk', version: graalvmVersion
+  compileOnly group: 'org.graalvm.nativeimage', name: 'svm', version: graalvmVersion
+
+  implementation "org.scala-lang:scala-library:${scalaVersion}"
+
+  testImplementation platform('org.junit:junit-bom:5.10.0')
+  testImplementation 'org.junit.jupiter:junit-jupiter'
+  testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
+}
+
+test {
+  useJUnitPlatform()
+}
+
+tasks.matching { it.name == 'nativeCompileClasspathJar' }.configureEach { t ->
+  t.exclude('META-INF/services/org.mule.weave.v2.module.DataFormat')
+  t.from("${projectDir}/src/main/resources/META-INF/services/org.mule.weave.v2.module.DataFormat") {
+    into('META-INF/services')
+    rename { 'org.mule.weave.v2.module.DataFormat' }
+  }
+}
+
+// Configure GraalVM native-image to build a shared library
+graalvmNative {
+//  toolchainDetection = true
+  binaries {
+    main {
+      sharedLibrary = true
+      debug = true
+      verbose = true
+      fallback = false
+      //agent = false
+      useFatJar = true
+      //buildArgs.add('-Ob') //  quick build mode to speed up builds during development
+      buildArgs.add('--no-fallback')
+      buildArgs.add('-H:Name=dwlib')
+      buildArgs.add('--verbose')
+      buildArgs.add("--report-unsupported-elements-at-runtime")
+      buildArgs.add("-J-Xmx6G")
+
+      buildArgs.add("-H:+ReportExceptionStackTraces")
+      buildArgs.add("-H:+UnlockExperimentalVMOptions")
+      buildArgs.add("--initialize-at-build-time=sun.instrument.InstrumentationImpl")
+      buildArgs.add("-H:DeadlockWatchdogInterval=1000")
+      buildArgs.add("-H:CompilationExpirationPeriod=0")
+      buildArgs.add("-H:+AddAllCharsets")
+      buildArgs.add("-H:+IncludeAllLocales")
+      // Pass project directory as system property for header path resolution
+      buildArgs.add("-Dproject.root=${projectDir}")
+    }
+  }
+}
+
+def pythonExe = (project.findProperty('pythonExe') ?: 'python3') as String
+
+tasks.register('stagePythonNativeLib', Copy) {
+  dependsOn tasks.named('nativeCompile')
+  from("${buildDir}/native/nativeCompile") {
+    include('dwlib.*')
+  }
+  into("${projectDir}/python/src/dataweave/native")
+}
+
+tasks.register('buildPythonWheel', Exec) {
+  dependsOn tasks.named('stagePythonNativeLib')
+  workingDir("${projectDir}/python")
+  outputs.dir("${projectDir}/python/dist")
+  doFirst {
+    file("${projectDir}/python/dist").mkdirs()
+  }
+  commandLine(pythonExe, '-m', 'pip', 'wheel', '--no-deps', '-w', 'dist', '.')
+}
+
+tasks.register('pythonTest', Exec) {
+  if (project.findProperty('skipPythonTests')?.toString()?.toBoolean() == true) {
+    enabled = false
+  }
+
+  dependsOn tasks.named('stagePythonNativeLib')
+  workingDir("${projectDir}/python")
+  commandLine(pythonExe, 'tests/test_dataweave_module.py')
+}
+
+tasks.named('test') {
+  dependsOn tasks.named('pythonTest')
+}
+
+tasks.named('clean') {
+  delete("${projectDir}/python/dist")
+  delete("${projectDir}/python/src/dataweave/native")
+}