Fix DLL load failure on Windows , update workflow and simplify sdist packaging

.github/workflows/bld_wheels_and_upload.yml

@@ -21,12 +21,15 @@ jobs:
     steps:
       - uses: actions/checkout@v5
       - name: Build wheels
-        uses: pypa/cibuildwheel@v3.1.4
+        uses: pypa/cibuildwheel@v3.4.1
         env:
           CIBW_BUILD: "*-win_amd64"
           CIBW_SKIP: "cp38-*"
 
-      - uses: actions/upload-artifact@v4.4.3
+      - name: Inject ibm_db_dll.pth into wheels
+        run: python scripts/inject_pth_into_wheel.py wheelhouse
+
+      - uses: actions/upload-artifact@v6
         with:
           name: ibmdb-wheels64-${{ matrix.os }}
           path: wheelhouse/*.whl
@@ -46,7 +49,10 @@ jobs:
           CIBW_BUILD: "*-win32"
           CIBW_SKIP: "cp38-*"
 
-      - uses: actions/upload-artifact@v4.4.3
+      - name: Inject ibm_db_dll.pth into wheels
+        run: python scripts/inject_pth_into_wheel.py wheelhouse
+
+      - uses: actions/upload-artifact@v6
         with:
           name: ibmdb-wheels32-${{ matrix.os }}
           path: wheelhouse/*.whl
@@ -61,7 +67,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
       - name: Build wheels
-        uses: pypa/cibuildwheel@v3.1.4
+        uses: pypa/cibuildwheel@v3.4.1
         env:
          CIBW_ARCHS_LINUX: "x86_64 i686"
          CIBW_MANYLINUX_I686_IMAGE: manylinux2014
@@ -81,7 +87,7 @@ jobs:
               --wheel-dir {dest_dir}
               {wheel}
 
-      - uses: actions/upload-artifact@v4.4.3
+      - uses: actions/upload-artifact@v6
         with:
           name: ibmdb-wheels-${{ matrix.os }}
           path: wheelhouse/*.whl
@@ -96,12 +102,12 @@ jobs:
     steps:
       - uses: actions/checkout@v5
       - name: Build wheels
-        uses: pypa/cibuildwheel@v3.1.4
+        uses: pypa/cibuildwheel@v3.4.1
         env:
           CIBW_SKIP: "cp38-*"
           MACOSX_DEPLOYMENT_TARGET: 14.0
 
-      - uses: actions/upload-artifact@v4.4.3
+      - uses: actions/upload-artifact@v6
         with:
           name: ibmdb-wheelsarm64
           path: wheelhouse/*.whl
@@ -116,13 +122,13 @@ jobs:
     steps:
       - uses: actions/checkout@v5
       - name: Build wheels
-        uses: pypa/cibuildwheel@v3.1.4
+        uses: pypa/cibuildwheel@v3.4.1
         env:
           CIBW_ARCHS: "x86_64"
           CIBW_SKIP: "cp38-*"
           MACOSX_DEPLOYMENT_TARGET: 10.15
 
-      - uses: actions/upload-artifact@v4.4.3
+      - uses: actions/upload-artifact@v6
         with:
           name: ibmdb-wheelsx86-${{ matrix.os }}
           path: wheelhouse/*.whl
@@ -136,26 +142,19 @@ jobs:
         run: python -m pip install --upgrade pip build
       - name: Build sdist
         run: python -m build --sdist --no-isolation
-      - name: Package version 
-        id: version
+      - name: Remove clidriver from sdist
         run: |
-           cd dist
-           pip install ibm_db*
-           echo "VERSION=$(python -c 'import ibm_db; print(ibm_db.__version__)')" >> $GITHUB_OUTPUT
-      - name: Build source distribution
-        run: |
-          PACKAGE="ibm_db-$VERSION"
           cd dist
-          tar -xzf $PACKAGE.tar.gz
-          rm -rf $PACKAGE/clidriver*
-          rm -rf $PACKAGE.tar.gz
-          tar -czf $PACKAGE.tar.gz $PACKAGE
-          rm -rf $PACKAGE
-        env:
-          VERSION: ${{ steps.version.outputs.VERSION}}
+          TARBALL=$(ls ibm?db-*.tar.gz | head -1)
+          DIRNAME="${TARBALL%.tar.gz}"
+          tar -xzf "$TARBALL"
+          rm -rf "$DIRNAME"/clidriver*
+          rm -rf "$TARBALL"
+          tar -czf "$TARBALL" "$DIRNAME"
+          rm -rf "$DIRNAME"
 
       - name: Upload sdist
-        uses: actions/upload-artifact@v4.4.3
+        uses: actions/upload-artifact@v6
         with:
           name: ibmdb-sdist
           path: dist/*.tar.gz
@@ -173,12 +172,12 @@ jobs:
     #upload to PyPI on every tag starting with 'v'
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
     steps:
-    - uses: actions/download-artifact@v5
+    - uses: actions/download-artifact@v7
       with:
         path: dist
         pattern: ibmdb-*
         merge-multiple: true
 
     - name: Publish distribution to PyPI
       if: startsWith(github.ref, 'refs/tags')
-      uses: pypa/gh-action-pypi-publish@release/v1.12
+      uses: pypa/gh-action-pypi-publish@release/v1.13

MANIFEST.in

@@ -1,6 +1,8 @@
 include ibm_db.h ibm_db.c
 include CHANGES.md LICENSE README.md
 include config.py.sample
+include ibm_db_dll.pth
+include _ibm_db_register_dll.py
 recursive-include ibm_db_tests *.py *.png *.jpg
 include MANIFEST.in
 recursive-include clidriver *

README.md

@@ -119,31 +119,38 @@ pip install ibm_db --no-binary :all: --no-cache-dir
 
 - When ibm_db get installed from wheel package, you can find clidriver under site_packages directory of Python. You need to copy license file under `site_packages/clidriver/license` to be effective, if any.
 
-**Note:** For windows after installing ibm_db, recieves the below error when we try to import ibm_db :
+**Windows DLL resolution (Python 3.8+):**
+
+Since Python 3.8, the `PATH` environment variable is no longer used for DLL resolution on Windows (see https://bugs.python.org/issue36085). The `ibm_db` package now handles this **automatically** by installing an `ibm_db_dll.pth` file into `site-packages`. This file runs at Python startup and registers the clidriver `bin` directory via `os.add_dll_directory()`, so `import ibm_db` works out of the box.
+
+If `IBM_DB_HOME` is set, the `.pth` file uses `%IBM_DB_HOME%\bin`; otherwise it uses the bundled `site-packages\clidriver\bin`.
+
+**If you still see `ImportError: DLL load failed` after a fresh install**, verify that the `.pth` file exists:
 
-```>python
-Python 3.11.4 (tags/v3.11.4:d2340ef, Jun  7 2023, 05:45:37) [MSC v.1934 64 bit (AMD64)] on win32
-Type "help", "copyright", "credits" or "license" for more information.
->>> import ibm_db
-Traceback (most recent call last):
-  File "<stdin>", line 1, in <module>
-ImportError: DLL load failed while importing ibm_db: The specified module could not be found.
->>>
+```
+python -c "import os, sysconfig; print(os.path.isfile(os.path.join(sysconfig.get_path('purelib'), 'ibm_db_dll.pth')))"
 ```
 
-We need to make sure to set dll path of dependent library of clidriver before importing the module as:
+If it prints `False`, reinstall ibm_db:
 
 ```
+pip uninstall ibm_db
+pip install ibm_db
+```
+
+**Manual fallback:** If the automatic fix does not work in your environment, you can set the DLL path directly in your code before importing the module:
+
+```python
 import os
 os.add_dll_directory('path to clidriver installation until bin')
 import ibm_db
-
-e.g:
-os.add_dll_directory('C:\\Program Files\\IBM\\CLIDRIVER\\bin')
-import ibm_db
 ```
 
-Refer https://bugs.python.org/issue36085 for more details.
+To find your clidriver `bin` path, run:
+
+```
+python -c "import os, site, sysconfig; paths=[os.path.join(site.getusersitepackages(),'clidriver','bin'), os.path.join(sysconfig.get_path('purelib'),'clidriver','bin')]; print(next((p for p in paths if os.path.isdir(p)), 'clidriver not found - reinstall ibm_db'))"
+```
 
 - <a name="docker"></a>For installing ibm_db on docker Linux container, you can refer as below:
 

_ibm_db_register_dll.py

@@ -0,0 +1,38 @@
+# Auto-generated by ibm_db setup.py
+# Registers clidriver DLL directory for Python 3.8+ on Windows.
+# This module is imported at startup via ibm_db_dll.pth.
+import os, sys, site, sysconfig
+
+if sys.platform == "win32" and hasattr(os, "add_dll_directory"):
+    candidates = []
+
+    # 1. IBM_DB_HOME environment variable (highest priority)
+    ibm_home = os.environ.get("IBM_DB_HOME")
+    if ibm_home:
+        candidates.append(os.path.join(ibm_home.strip('"'), "bin"))
+    else:
+        # 2. User site-packages/clidriver (pip install --user)
+        try:
+            usp = site.getusersitepackages()
+            if usp:
+                candidates.append(os.path.join(usp, "clidriver", "bin"))
+        except Exception:
+            pass
+
+        # 3. System site-packages/clidriver (standard pip install)
+        candidates.append(
+            os.path.join(sysconfig.get_path("purelib"), "clidriver", "bin")
+        )
+
+        # 4. PATH entries that look like DB2/clidriver installs
+        for d in os.environ.get("PATH", "").split(";"):
+            if d and os.path.basename(d).lower() == "bin":
+                if (os.path.isfile(os.path.join(d, "db2cli.exe")) or
+                        os.path.isdir(os.path.join(os.path.dirname(d), "license"))):
+                    candidates.append(d)
+
+    # Register the first valid DLL directory
+    for p in candidates:
+        if p and os.path.isdir(p):
+            os.add_dll_directory(p)
+            break

ibm_db.c

@@ -2354,8 +2354,8 @@ static PyObject *_python_ibm_db_connect_helper(PyObject *self, PyObject *args, i
                                        database, SQL_NTS, NULL, 0, NULL,
                                        SQL_DRIVER_NOPROMPT);
                 Py_END_ALLOW_THREADS;
-                snprintf(messageStr, sizeof(messageStr), "SQLDriverConnectW called with parameters: conn_res->hdbc=%p, SQLHWND=NULL, database=%ls, SQL_NTS=%d, NULL, 0, NULL, SQL_DRIVER_NOPROMPT=%d and returned rc=%d",
-                         (void *)conn_res->hdbc, database, SQL_NTS, SQL_DRIVER_NOPROMPT, rc);
+                snprintf(messageStr, sizeof(messageStr), "SQLDriverConnectW called with parameters: conn_res->hdbc=%p, SQLHWND=NULL, database=%s, SQL_NTS=%d, NULL, 0, NULL, SQL_DRIVER_NOPROMPT=%d and returned rc=%d",
+                         (void *)conn_res->hdbc, PyUnicode_AsUTF8(databaseObj), SQL_NTS, SQL_DRIVER_NOPROMPT, rc);
                 LogMsg(DEBUG, messageStr);
             }
             else
@@ -2378,10 +2378,10 @@ static PyObject *_python_ibm_db_connect_helper(PyObject *self, PyObject *args, i
                                  PyUnicode_GetLength(uidObj) * 2,
                                  password,
                                  PyUnicode_GetLength(passwordObj) * 2);
-                snprintf(messageStr, sizeof(messageStr), "SQLConnectW called with parameters: conn_res->hdbc=%p, database=%ls, databaseLen=%zd, uid=%ls, uidLen=%zd, password=%ls, passwordLen=%zd and returned rc=%d",
-                         (void *)conn_res->hdbc, database,
-                         PyUnicode_GetLength(databaseObj) * 2, uid,
-                         PyUnicode_GetLength(uidObj) * 2, password,
+			    snprintf(messageStr, sizeof(messageStr), "SQLConnectW called with parameters: conn_res->hdbc=%p, database=%s, databaseLen=%zd, uid=%s, uidLen=%zd, password=%s, passwordLen=%zd and returned rc=%d",
+                         (void *)conn_res->hdbc, PyUnicode_AsUTF8(databaseObj),
+                         PyUnicode_GetLength(databaseObj) * 2, PyUnicode_AsUTF8(uidObj),
+                         PyUnicode_GetLength(uidObj) * 2, PyUnicode_AsUTF8(passwordObj),
                          PyUnicode_GetLength(passwordObj) * 2, rc);
                 LogMsg(DEBUG, messageStr);
 #else
@@ -2392,10 +2392,10 @@ static PyObject *_python_ibm_db_connect_helper(PyObject *self, PyObject *args, i
                                  PyUnicode_GetLength(uidObj),
                                  password,
                                  PyUnicode_GetLength(passwordObj));
-                snprintf(messageStr, sizeof(messageStr), "SQLConnectW called with parameters: conn_res->hdbc=%p, database=%ls, databaseLen=%zd, uid=%ls, uidLen=%zd, password=%ls, passwordLen=%zd and returned rc=%d",
-                         (void *)conn_res->hdbc, database,
-                         PyUnicode_GetLength(databaseObj), uid,
-                         PyUnicode_GetLength(uidObj), password,
+                snprintf(messageStr, sizeof(messageStr), "SQLConnectW called with parameters: conn_res->hdbc=%p, database=%s, databaseLen=%zd, uid=%s, uidLen=%zd, password=%s, passwordLen=%zd and returned rc=%d",
+                         (void *)conn_res->hdbc, PyUnicode_AsUTF8(databaseObj),
+                         PyUnicode_GetLength(databaseObj), PyUnicode_AsUTF8(uidObj),
+                         PyUnicode_GetLength(uidObj), PyUnicode_AsUTF8(passwordObj),
                          PyUnicode_GetLength(passwordObj), rc);
                 LogMsg(DEBUG, messageStr);
 #endif

ibm_db_dll.pth

@@ -0,0 +1 @@
+import _ibm_db_register_dll

scripts/inject_pth_into_wheel.py

@@ -0,0 +1,80 @@
+"""Inject ibm_db_dll.pth into a wheel so it lands in site-packages on install.
+
+Usage: python scripts/inject_pth_into_wheel.py <wheel_dir>
+
+Wheels are zip files. Files at the root level of a wheel (alongside .py
+modules) are installed to site-packages. This script adds ibm_db_dll.pth
+to every .whl file in the given directory.
+"""
+import os, sys, hashlib, base64, zipfile, glob, tempfile, shutil
+
+PTH_FILENAME = 'ibm_db_dll.pth'
+PTH_CONTENT = 'import _ibm_db_register_dll\n'
+
+
+def _record_line(name, content_bytes):
+    """Build a RECORD entry: name,sha256=<digest>,<length>"""
+    digest = hashlib.sha256(content_bytes).digest()
+    b64 = base64.urlsafe_b64encode(digest).rstrip(b'=').decode('ascii')
+    return f'{name},sha256={b64},{len(content_bytes)}'
+
+
+def inject_pth(whl_path):
+    """Add ibm_db_dll.pth to a wheel file and remove any misplaced copies."""
+    with zipfile.ZipFile(whl_path, 'r') as zin:
+        names = zin.namelist()
+        # Skip if the .pth file is already at the wheel root
+        if PTH_FILENAME in names:
+            print(f'  {PTH_FILENAME} already at root of {os.path.basename(whl_path)}, skipping')
+            return
+
+    tmp_fd, tmp_path = tempfile.mkstemp(suffix='.whl')
+    os.close(tmp_fd)
+
+    pth_bytes = PTH_CONTENT.encode('utf-8')
+    pth_record = _record_line(PTH_FILENAME, pth_bytes)
+
+    with zipfile.ZipFile(whl_path, 'r') as zin, \
+         zipfile.ZipFile(tmp_path, 'w', zipfile.ZIP_DEFLATED) as zout:
+
+        for item in zin.infolist():
+            # Drop any misplaced copies of the .pth file (absolute-path junk from data_files)
+            if item.filename != PTH_FILENAME and item.filename.endswith('/' + PTH_FILENAME):
+                print(f'  Removing misplaced {item.filename}')
+                continue
+
+            data = zin.read(item.filename)
+
+            # Append our .pth entry to the RECORD file
+            if item.filename.endswith('/RECORD'):
+                data = data.rstrip(b'\n') + b'\n' + pth_record.encode('utf-8') + b'\n'
+
+            zout.writestr(item, data)
+
+        # Add the .pth file at the wheel root
+        zout.writestr(PTH_FILENAME, pth_bytes)
+
+    shutil.move(tmp_path, whl_path)
+    print(f'  Injected {PTH_FILENAME} into {os.path.basename(whl_path)}')
+
+
+def main():
+    if len(sys.argv) != 2:
+        print(f'Usage: {sys.argv[0]} <wheel_dir>')
+        sys.exit(1)
+
+    wheel_dir = sys.argv[1]
+    wheels = glob.glob(os.path.join(wheel_dir, '*.whl'))
+
+    if not wheels:
+        print(f'No .whl files found in {wheel_dir}')
+        sys.exit(1)
+
+    for whl in wheels:
+        inject_pth(whl)
+
+    print(f'Done: processed {len(wheels)} wheel(s)')
+
+
+if __name__ == '__main__':
+    main()

setup.py

@@ -28,6 +28,7 @@
     from distutils.sysconfig import get_python_lib
 from setuptools import setup, find_packages, Extension
 from setuptools.command.build_ext import build_ext
+from setuptools.command.build_py import build_py
 from setuptools.command.install import install
 
 PACKAGE = 'ibm_db'
@@ -501,7 +502,7 @@ def print_exception( e, url):
                (get_python_lib(), ['./LICENSE']),
                (get_python_lib(), ['./config.py.sample'])]
 
-modules = ['ibm_db_dbi', 'testfunctions', 'ibmdb_tests', 'ibm_db_ctx']
+modules = ['ibm_db_dbi', 'testfunctions', 'ibmdb_tests', 'ibm_db_ctx', '_ibm_db_register_dll']
 
 if 'zos' == sys.platform:
     ext_modules = _ext_modules(os.path.join(os.getcwd(), include_dir), library, ibm_db_lib, ibm_db_lib_runtime)
@@ -525,6 +526,18 @@ def print_exception( e, url):
            _checkGcc()
            _checkPythonHeaderFile()
 
+# Custom build_py to include ibm_db_dll.pth at the wheel root
+# so it lands in site-packages and triggers DLL registration on startup.
+class _build_py_with_pth(build_py):
+    def run(self):
+        super().run()
+        pth_src = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'ibm_db_dll.pth')
+        if os.path.isfile(pth_src):
+            pth_dst = os.path.join(self.build_lib, 'ibm_db_dll.pth')
+            self.copy_file(pth_src, pth_dst)
+
+cmd_class['build_py'] = _build_py_with_pth
+
 #'Operating System :: z/OS', pypi upload fails with error - Not a valid classifier
 setup( name    = PACKAGE,
     version = VERSION,