diff --git a/.github/workflows/bld_wheels_and_upload.yml b/.github/workflows/bld_wheels_and_upload.yml index 5c968ebe..9f5bc9ec 100644 --- a/.github/workflows/bld_wheels_and_upload.yml +++ b/.github/workflows/bld_wheels_and_upload.yml @@ -21,12 +21,15 @@ jobs: steps: - uses: actions/checkout@v5 - name: Build wheels - uses: pypa/cibuildwheel@v3.1.4 + uses: pypa/cibuildwheel@v3.4.1 env: CIBW_BUILD: "*-win_amd64" CIBW_SKIP: "cp38-*" - - uses: actions/upload-artifact@v4.4.3 + - name: Inject ibm_db_dll.pth into wheels + run: python scripts/inject_pth_into_wheel.py wheelhouse + + - uses: actions/upload-artifact@v6 with: name: ibmdb-wheels64-${{ matrix.os }} path: wheelhouse/*.whl @@ -46,7 +49,10 @@ jobs: CIBW_BUILD: "*-win32" CIBW_SKIP: "cp38-*" - - uses: actions/upload-artifact@v4.4.3 + - name: Inject ibm_db_dll.pth into wheels + run: python scripts/inject_pth_into_wheel.py wheelhouse + + - uses: actions/upload-artifact@v6 with: name: ibmdb-wheels32-${{ matrix.os }} path: wheelhouse/*.whl @@ -61,7 +67,7 @@ jobs: steps: - uses: actions/checkout@v5 - name: Build wheels - uses: pypa/cibuildwheel@v3.1.4 + uses: pypa/cibuildwheel@v3.4.1 env: CIBW_ARCHS_LINUX: "x86_64 i686" CIBW_MANYLINUX_I686_IMAGE: manylinux2014 @@ -81,7 +87,7 @@ jobs: --wheel-dir {dest_dir} {wheel} - - uses: actions/upload-artifact@v4.4.3 + - uses: actions/upload-artifact@v6 with: name: ibmdb-wheels-${{ matrix.os }} path: wheelhouse/*.whl @@ -96,12 +102,12 @@ jobs: steps: - uses: actions/checkout@v5 - name: Build wheels - uses: pypa/cibuildwheel@v3.1.4 + uses: pypa/cibuildwheel@v3.4.1 env: CIBW_SKIP: "cp38-*" MACOSX_DEPLOYMENT_TARGET: 14.0 - - uses: actions/upload-artifact@v4.4.3 + - uses: actions/upload-artifact@v6 with: name: ibmdb-wheelsarm64 path: wheelhouse/*.whl @@ -116,13 +122,13 @@ jobs: steps: - uses: actions/checkout@v5 - name: Build wheels - uses: pypa/cibuildwheel@v3.1.4 + uses: pypa/cibuildwheel@v3.4.1 env: CIBW_ARCHS: "x86_64" CIBW_SKIP: "cp38-*" MACOSX_DEPLOYMENT_TARGET: 10.15 - - uses: actions/upload-artifact@v4.4.3 + - uses: actions/upload-artifact@v6 with: name: ibmdb-wheelsx86-${{ matrix.os }} path: wheelhouse/*.whl @@ -136,26 +142,19 @@ jobs: run: python -m pip install --upgrade pip build - name: Build sdist run: python -m build --sdist --no-isolation - - name: Package version - id: version + - name: Remove clidriver from sdist run: | - cd dist - pip install ibm_db* - echo "VERSION=$(python -c 'import ibm_db; print(ibm_db.__version__)')" >> $GITHUB_OUTPUT - - name: Build source distribution - run: | - PACKAGE="ibm_db-$VERSION" cd dist - tar -xzf $PACKAGE.tar.gz - rm -rf $PACKAGE/clidriver* - rm -rf $PACKAGE.tar.gz - tar -czf $PACKAGE.tar.gz $PACKAGE - rm -rf $PACKAGE - env: - VERSION: ${{ steps.version.outputs.VERSION}} + TARBALL=$(ls ibm?db-*.tar.gz | head -1) + DIRNAME="${TARBALL%.tar.gz}" + tar -xzf "$TARBALL" + rm -rf "$DIRNAME"/clidriver* + rm -rf "$TARBALL" + tar -czf "$TARBALL" "$DIRNAME" + rm -rf "$DIRNAME" - name: Upload sdist - uses: actions/upload-artifact@v4.4.3 + uses: actions/upload-artifact@v6 with: name: ibmdb-sdist path: dist/*.tar.gz @@ -173,7 +172,7 @@ jobs: #upload to PyPI on every tag starting with 'v' if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') steps: - - uses: actions/download-artifact@v5 + - uses: actions/download-artifact@v7 with: path: dist pattern: ibmdb-* @@ -181,4 +180,4 @@ jobs: - name: Publish distribution to PyPI if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@release/v1.12 + uses: pypa/gh-action-pypi-publish@release/v1.13 diff --git a/MANIFEST.in b/MANIFEST.in index 9f349ba4..8dd2ba3a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,8 @@ include ibm_db.h ibm_db.c include CHANGES.md LICENSE README.md include config.py.sample +include ibm_db_dll.pth +include _ibm_db_register_dll.py recursive-include ibm_db_tests *.py *.png *.jpg include MANIFEST.in recursive-include clidriver * diff --git a/README.md b/README.md index c843f39d..9429e678 100644 --- a/README.md +++ b/README.md @@ -119,31 +119,38 @@ pip install ibm_db --no-binary :all: --no-cache-dir - When ibm_db get installed from wheel package, you can find clidriver under site_packages directory of Python. You need to copy license file under `site_packages/clidriver/license` to be effective, if any. -**Note:** For windows after installing ibm_db, recieves the below error when we try to import ibm_db : +**Windows DLL resolution (Python 3.8+):** + +Since Python 3.8, the `PATH` environment variable is no longer used for DLL resolution on Windows (see https://bugs.python.org/issue36085). The `ibm_db` package now handles this **automatically** by installing an `ibm_db_dll.pth` file into `site-packages`. This file runs at Python startup and registers the clidriver `bin` directory via `os.add_dll_directory()`, so `import ibm_db` works out of the box. + +If `IBM_DB_HOME` is set, the `.pth` file uses `%IBM_DB_HOME%\bin`; otherwise it uses the bundled `site-packages\clidriver\bin`. + +**If you still see `ImportError: DLL load failed` after a fresh install**, verify that the `.pth` file exists: -```>python -Python 3.11.4 (tags/v3.11.4:d2340ef, Jun 7 2023, 05:45:37) [MSC v.1934 64 bit (AMD64)] on win32 -Type "help", "copyright", "credits" or "license" for more information. ->>> import ibm_db -Traceback (most recent call last): - File "", line 1, in -ImportError: DLL load failed while importing ibm_db: The specified module could not be found. ->>> +``` +python -c "import os, sysconfig; print(os.path.isfile(os.path.join(sysconfig.get_path('purelib'), 'ibm_db_dll.pth')))" ``` -We need to make sure to set dll path of dependent library of clidriver before importing the module as: +If it prints `False`, reinstall ibm_db: ``` +pip uninstall ibm_db +pip install ibm_db +``` + +**Manual fallback:** If the automatic fix does not work in your environment, you can set the DLL path directly in your code before importing the module: + +```python import os os.add_dll_directory('path to clidriver installation until bin') import ibm_db - -e.g: -os.add_dll_directory('C:\\Program Files\\IBM\\CLIDRIVER\\bin') -import ibm_db ``` -Refer https://bugs.python.org/issue36085 for more details. +To find your clidriver `bin` path, run: + +``` +python -c "import os, site, sysconfig; paths=[os.path.join(site.getusersitepackages(),'clidriver','bin'), os.path.join(sysconfig.get_path('purelib'),'clidriver','bin')]; print(next((p for p in paths if os.path.isdir(p)), 'clidriver not found - reinstall ibm_db'))" +``` - For installing ibm_db on docker Linux container, you can refer as below: diff --git a/_ibm_db_register_dll.py b/_ibm_db_register_dll.py new file mode 100644 index 00000000..3ea0b189 --- /dev/null +++ b/_ibm_db_register_dll.py @@ -0,0 +1,38 @@ +# Auto-generated by ibm_db setup.py +# Registers clidriver DLL directory for Python 3.8+ on Windows. +# This module is imported at startup via ibm_db_dll.pth. +import os, sys, site, sysconfig + +if sys.platform == "win32" and hasattr(os, "add_dll_directory"): + candidates = [] + + # 1. IBM_DB_HOME environment variable (highest priority) + ibm_home = os.environ.get("IBM_DB_HOME") + if ibm_home: + candidates.append(os.path.join(ibm_home.strip('"'), "bin")) + else: + # 2. User site-packages/clidriver (pip install --user) + try: + usp = site.getusersitepackages() + if usp: + candidates.append(os.path.join(usp, "clidriver", "bin")) + except Exception: + pass + + # 3. System site-packages/clidriver (standard pip install) + candidates.append( + os.path.join(sysconfig.get_path("purelib"), "clidriver", "bin") + ) + + # 4. PATH entries that look like DB2/clidriver installs + for d in os.environ.get("PATH", "").split(";"): + if d and os.path.basename(d).lower() == "bin": + if (os.path.isfile(os.path.join(d, "db2cli.exe")) or + os.path.isdir(os.path.join(os.path.dirname(d), "license"))): + candidates.append(d) + + # Register the first valid DLL directory + for p in candidates: + if p and os.path.isdir(p): + os.add_dll_directory(p) + break diff --git a/ibm_db.c b/ibm_db.c index 5fd94a82..a702f32e 100644 --- a/ibm_db.c +++ b/ibm_db.c @@ -2354,8 +2354,8 @@ static PyObject *_python_ibm_db_connect_helper(PyObject *self, PyObject *args, i database, SQL_NTS, NULL, 0, NULL, SQL_DRIVER_NOPROMPT); Py_END_ALLOW_THREADS; - snprintf(messageStr, sizeof(messageStr), "SQLDriverConnectW called with parameters: conn_res->hdbc=%p, SQLHWND=NULL, database=%ls, SQL_NTS=%d, NULL, 0, NULL, SQL_DRIVER_NOPROMPT=%d and returned rc=%d", - (void *)conn_res->hdbc, database, SQL_NTS, SQL_DRIVER_NOPROMPT, rc); + snprintf(messageStr, sizeof(messageStr), "SQLDriverConnectW called with parameters: conn_res->hdbc=%p, SQLHWND=NULL, database=%s, SQL_NTS=%d, NULL, 0, NULL, SQL_DRIVER_NOPROMPT=%d and returned rc=%d", + (void *)conn_res->hdbc, PyUnicode_AsUTF8(databaseObj), SQL_NTS, SQL_DRIVER_NOPROMPT, rc); LogMsg(DEBUG, messageStr); } else @@ -2378,10 +2378,10 @@ static PyObject *_python_ibm_db_connect_helper(PyObject *self, PyObject *args, i PyUnicode_GetLength(uidObj) * 2, password, PyUnicode_GetLength(passwordObj) * 2); - snprintf(messageStr, sizeof(messageStr), "SQLConnectW called with parameters: conn_res->hdbc=%p, database=%ls, databaseLen=%zd, uid=%ls, uidLen=%zd, password=%ls, passwordLen=%zd and returned rc=%d", - (void *)conn_res->hdbc, database, - PyUnicode_GetLength(databaseObj) * 2, uid, - PyUnicode_GetLength(uidObj) * 2, password, + snprintf(messageStr, sizeof(messageStr), "SQLConnectW called with parameters: conn_res->hdbc=%p, database=%s, databaseLen=%zd, uid=%s, uidLen=%zd, password=%s, passwordLen=%zd and returned rc=%d", + (void *)conn_res->hdbc, PyUnicode_AsUTF8(databaseObj), + PyUnicode_GetLength(databaseObj) * 2, PyUnicode_AsUTF8(uidObj), + PyUnicode_GetLength(uidObj) * 2, PyUnicode_AsUTF8(passwordObj), PyUnicode_GetLength(passwordObj) * 2, rc); LogMsg(DEBUG, messageStr); #else @@ -2392,10 +2392,10 @@ static PyObject *_python_ibm_db_connect_helper(PyObject *self, PyObject *args, i PyUnicode_GetLength(uidObj), password, PyUnicode_GetLength(passwordObj)); - snprintf(messageStr, sizeof(messageStr), "SQLConnectW called with parameters: conn_res->hdbc=%p, database=%ls, databaseLen=%zd, uid=%ls, uidLen=%zd, password=%ls, passwordLen=%zd and returned rc=%d", - (void *)conn_res->hdbc, database, - PyUnicode_GetLength(databaseObj), uid, - PyUnicode_GetLength(uidObj), password, + snprintf(messageStr, sizeof(messageStr), "SQLConnectW called with parameters: conn_res->hdbc=%p, database=%s, databaseLen=%zd, uid=%s, uidLen=%zd, password=%s, passwordLen=%zd and returned rc=%d", + (void *)conn_res->hdbc, PyUnicode_AsUTF8(databaseObj), + PyUnicode_GetLength(databaseObj), PyUnicode_AsUTF8(uidObj), + PyUnicode_GetLength(uidObj), PyUnicode_AsUTF8(passwordObj), PyUnicode_GetLength(passwordObj), rc); LogMsg(DEBUG, messageStr); #endif diff --git a/ibm_db_dll.pth b/ibm_db_dll.pth new file mode 100644 index 00000000..6119985f --- /dev/null +++ b/ibm_db_dll.pth @@ -0,0 +1 @@ +import _ibm_db_register_dll diff --git a/scripts/inject_pth_into_wheel.py b/scripts/inject_pth_into_wheel.py new file mode 100644 index 00000000..544969cd --- /dev/null +++ b/scripts/inject_pth_into_wheel.py @@ -0,0 +1,80 @@ +"""Inject ibm_db_dll.pth into a wheel so it lands in site-packages on install. + +Usage: python scripts/inject_pth_into_wheel.py + +Wheels are zip files. Files at the root level of a wheel (alongside .py +modules) are installed to site-packages. This script adds ibm_db_dll.pth +to every .whl file in the given directory. +""" +import os, sys, hashlib, base64, zipfile, glob, tempfile, shutil + +PTH_FILENAME = 'ibm_db_dll.pth' +PTH_CONTENT = 'import _ibm_db_register_dll\n' + + +def _record_line(name, content_bytes): + """Build a RECORD entry: name,sha256=,""" + digest = hashlib.sha256(content_bytes).digest() + b64 = base64.urlsafe_b64encode(digest).rstrip(b'=').decode('ascii') + return f'{name},sha256={b64},{len(content_bytes)}' + + +def inject_pth(whl_path): + """Add ibm_db_dll.pth to a wheel file and remove any misplaced copies.""" + with zipfile.ZipFile(whl_path, 'r') as zin: + names = zin.namelist() + # Skip if the .pth file is already at the wheel root + if PTH_FILENAME in names: + print(f' {PTH_FILENAME} already at root of {os.path.basename(whl_path)}, skipping') + return + + tmp_fd, tmp_path = tempfile.mkstemp(suffix='.whl') + os.close(tmp_fd) + + pth_bytes = PTH_CONTENT.encode('utf-8') + pth_record = _record_line(PTH_FILENAME, pth_bytes) + + with zipfile.ZipFile(whl_path, 'r') as zin, \ + zipfile.ZipFile(tmp_path, 'w', zipfile.ZIP_DEFLATED) as zout: + + for item in zin.infolist(): + # Drop any misplaced copies of the .pth file (absolute-path junk from data_files) + if item.filename != PTH_FILENAME and item.filename.endswith('/' + PTH_FILENAME): + print(f' Removing misplaced {item.filename}') + continue + + data = zin.read(item.filename) + + # Append our .pth entry to the RECORD file + if item.filename.endswith('/RECORD'): + data = data.rstrip(b'\n') + b'\n' + pth_record.encode('utf-8') + b'\n' + + zout.writestr(item, data) + + # Add the .pth file at the wheel root + zout.writestr(PTH_FILENAME, pth_bytes) + + shutil.move(tmp_path, whl_path) + print(f' Injected {PTH_FILENAME} into {os.path.basename(whl_path)}') + + +def main(): + if len(sys.argv) != 2: + print(f'Usage: {sys.argv[0]} ') + sys.exit(1) + + wheel_dir = sys.argv[1] + wheels = glob.glob(os.path.join(wheel_dir, '*.whl')) + + if not wheels: + print(f'No .whl files found in {wheel_dir}') + sys.exit(1) + + for whl in wheels: + inject_pth(whl) + + print(f'Done: processed {len(wheels)} wheel(s)') + + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index 9a7f4447..b0c5487d 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ from distutils.sysconfig import get_python_lib from setuptools import setup, find_packages, Extension from setuptools.command.build_ext import build_ext +from setuptools.command.build_py import build_py from setuptools.command.install import install PACKAGE = 'ibm_db' @@ -501,7 +502,7 @@ def print_exception( e, url): (get_python_lib(), ['./LICENSE']), (get_python_lib(), ['./config.py.sample'])] -modules = ['ibm_db_dbi', 'testfunctions', 'ibmdb_tests', 'ibm_db_ctx'] +modules = ['ibm_db_dbi', 'testfunctions', 'ibmdb_tests', 'ibm_db_ctx', '_ibm_db_register_dll'] if 'zos' == sys.platform: ext_modules = _ext_modules(os.path.join(os.getcwd(), include_dir), library, ibm_db_lib, ibm_db_lib_runtime) @@ -525,6 +526,18 @@ def print_exception( e, url): _checkGcc() _checkPythonHeaderFile() +# Custom build_py to include ibm_db_dll.pth at the wheel root +# so it lands in site-packages and triggers DLL registration on startup. +class _build_py_with_pth(build_py): + def run(self): + super().run() + pth_src = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'ibm_db_dll.pth') + if os.path.isfile(pth_src): + pth_dst = os.path.join(self.build_lib, 'ibm_db_dll.pth') + self.copy_file(pth_src, pth_dst) + +cmd_class['build_py'] = _build_py_with_pth + #'Operating System :: z/OS', pypi upload fails with error - Not a valid classifier setup( name = PACKAGE, version = VERSION,