Update Jenkins build to work with pyproject.toml

patch by Bret McGuire; reviewed by Bret McGuire and Brad Schoening

reference: #1271

Author: absurdfarce

Jenkinsfile

@@ -34,7 +34,7 @@ slack = new Slack()
 DEFAULT_CASSANDRA = ['3.11', '4.0', '4.1', '5.0']
 DEFAULT_DSE = ['dse-5.1.35', 'dse-6.8.30', 'dse-6.9.0']
 DEFAULT_HCD = ['hcd-1.0.0']
-DEFAULT_RUNTIME = ['3.9.23', '3.10.18', '3.11.13', '3.12.11', '3.13.5']
+DEFAULT_RUNTIME = ['3.10.18', '3.11.13', '3.12.11', '3.13.5']
 DEFAULT_CYTHON = ["True", "False"]
 matrices = [
   "FULL": [
@@ -119,7 +119,7 @@ def getBuildContext() {
 
 def buildAndTest(context) {
   initializeEnvironment()
-  installDriverAndCompileExtensions()
+  installDriver()
 
   try {
       executeTests()
@@ -165,15 +165,18 @@ def getMatrixBuilds(buildContext) {
 
 def initializeEnvironment() {
   sh label: 'Initialize the environment', script: '''#!/bin/bash -lex
-    pyenv global ${PYTHON_VERSION}
-    sudo apt-get install socat
-    pip install --upgrade pip
-    pip install -U setuptools
 
-    # install a version of pyyaml<6.0 compatible with ccm-3.1.5 as of Aug 2023
+    # One of the integration tests relies on socat so let's install that here
+    sudo apt-get install -y socat moreutils
+
+    pyenv shell ${PYTHON_VERSION}
+    python -m venv jenkins-venv
+    . ./jenkins-venv/bin/activate
+    pip install --upgrade pip setuptools wheel
+
+    # Install a version of pyyaml<6.0 compatible with ccm-3.1.5 as of Aug 2023
     # this works around the python-3.10+ compatibility problem as described in DSP-23524
-    pip install wheel
-    pip install "Cython<3.0" "pyyaml<6.0" --no-build-isolation
+    pip install "pyyaml<6.0" --no-build-isolation
     pip install ${HOME}/ccm
   '''
 
@@ -182,35 +185,28 @@ def initializeEnvironment() {
     if (env.PYTHON_VERSION =~ /3\.12\.\d+/) {
       echo "Cannot install DSE dependencies for Python 3.12.x; installing Apache CassandraⓇ requirements only.  See PYTHON-1368 for more detail."
       sh label: 'Install Apache CassandraⓇ requirements', script: '''#!/bin/bash -lex
+        . ./jenkins-venv/bin/activate
         pip install -r test-requirements.txt
       '''
     }
     else {
       sh label: 'Install DataStax Enterprise requirements', script: '''#!/bin/bash -lex
+        . ./jenkins-venv/bin/activate
         pip install -r test-datastax-requirements.txt
       '''
     }
   } else {
     sh label: 'Install Apache CassandraⓇ requirements', script: '''#!/bin/bash -lex
+      . ./jenkins-venv/bin/activate
       pip install -r test-requirements.txt
     '''
 
     sh label: 'Uninstall the geomet dependency since it is not required for Cassandra', script: '''#!/bin/bash -lex
+      . ./jenkins-venv/bin/activate
       pip uninstall -y geomet
     '''
   }
 
-  sh label: 'Install unit test modules', script: '''#!/bin/bash -lex
-    pip install --no-deps nose-ignore-docstring nose-exclude
-    pip install service_identity
-  '''
-
-  if (env.CYTHON_ENABLED  == 'True') {
-    sh label: 'Install cython modules', script: '''#!/bin/bash -lex
-      pip install cython numpy
-    '''
-  }
-
   sh label: 'Download Apache CassandraⓇ or DataStax Enterprise', script: '''#!/bin/bash -lex
     . ${CCM_ENVIRONMENT_SHELL} ${CASSANDRA_VERSION}
   '''
@@ -244,34 +240,55 @@ ENVIRONMENT_EOF
   }
 
   sh label: 'Display Python and environment information', script: '''#!/bin/bash -le
+    . ./jenkins-venv/bin/activate
+
     # Load CCM environment variables
     set -o allexport
     . ${HOME}/environment.txt
     set +o allexport
 
     python --version
     pip --version
-    pip freeze
     printenv | sort
   '''
 }
 
-def installDriverAndCompileExtensions() {
-  if (env.CYTHON_ENABLED  == 'True') {
-    sh label: 'Install the driver and compile with C extensions with Cython', script: '''#!/bin/bash -lex
-      python setup.py build_ext --inplace
-    '''
-  } else {
-    sh label: 'Install the driver and compile with C extensions without Cython', script: '''#!/bin/bash -lex
-      python setup.py build_ext --inplace --no-cython
-    '''
-  }
+def installDriver() {
+  sh label: 'Install the driver and compile with C extensions with Cython', script: '''#!/bin/bash -lex
+    # Update libev includes and libs to point to the right spot for this install
+    pyenv shell ${PYTHON_VERSION}
+    python -m venv libev-venv
+    . ./libev-venv/bin/activate
+    pip install toml
+    python fix-jenkinsfile-libev.py ./pyproject.toml "/usr/include" "/usr/lib/x86_64-linux-gnu" | sponge ./pyproject.toml
+    deactivate
+
+    ls /usr/include/ev.h
+    ls /usr/lib/x86_64-linux-gnu/libev*
+
+    # Now that we've made relevant mods to our local pyproject.toml we're ready to build the driver
+    . ./jenkins-venv/bin/activate
+
+    # Load CCM environment variables
+    set -o allexport
+    . ${HOME}/environment.txt
+    set +o allexport
+
+    cat ./pyproject.toml
+    pip install --verbose --editable .
+
+    # After install display a list of packages in the venv for auditing
+    pip list
+  '''
 }
 
+
 def executeStandardTests() {
 
   try {
     sh label: 'Execute unit tests', script: '''#!/bin/bash -lex
+      . ./jenkins-venv/bin/activate
+
       # Load CCM environment variables
       set -o allexport
       . ${HOME}/environment.txt
@@ -289,6 +306,8 @@ def executeStandardTests() {
 
   try {
     sh label: 'Execute Simulacron integration tests', script: '''#!/bin/bash -lex
+      . ./jenkins-venv/bin/activate
+
       # Load CCM environment variables
       set -o allexport
       . ${HOME}/environment.txt
@@ -314,6 +333,8 @@ def executeStandardTests() {
 
   try {
     sh label: 'Execute CQL engine integration tests', script: '''#!/bin/bash -lex
+      . ./jenkins-venv/bin/activate
+
       # Load CCM environment variables
       set -o allexport
       . ${HOME}/environment.txt
@@ -330,6 +351,8 @@ def executeStandardTests() {
 
   try {
     sh label: 'Execute Apache CassandraⓇ integration tests', script: '''#!/bin/bash -lex
+      . ./jenkins-venv/bin/activate
+
       # Load CCM environment variables
       set -o allexport
       . ${HOME}/environment.txt
@@ -351,6 +374,8 @@ def executeStandardTests() {
     else {
       try {
         sh label: 'Execute DataStax Enterprise integration tests', script: '''#!/bin/bash -lex
+          . ./jenkins-venv/bin/activate
+
           # Load CCM environment variable
           set -o allexport
           . ${HOME}/environment.txt
@@ -369,6 +394,8 @@ def executeStandardTests() {
 
   try {
     sh label: 'Execute DataStax Astra integration tests', script: '''#!/bin/bash -lex
+      . ./jenkins-venv/bin/activate
+
       # Load CCM environment variable
       set -o allexport
       . ${HOME}/environment.txt
@@ -386,6 +413,8 @@ def executeStandardTests() {
   if (env.PROFILE == 'FULL') {
     try {
       sh label: 'Execute long running integration tests', script: '''#!/bin/bash -lex
+        . ./jenkins-venv/bin/activate
+
         # Load CCM environment variable
         set -o allexport
         . ${HOME}/environment.txt
@@ -404,6 +433,8 @@ def executeStandardTests() {
 
 def executeDseSmokeTests() {
   sh label: 'Execute profile DataStax Enterprise smoke test integration tests', script: '''#!/bin/bash -lex
+    . ./jenkins-venv/bin/activate
+
     # Load CCM environment variable
     set -o allexport
     . ${HOME}/environment.txt
@@ -418,6 +449,8 @@ def executeDseSmokeTests() {
 
 def executeEventLoopTests() {
   sh label: 'Execute profile event loop manager integration tests', script: '''#!/bin/bash -lex
+    . ./jenkins-venv/bin/activate
+
     # Load CCM environment variable
     set -o allexport
     . ${HOME}/environment.txt

fix-jenkinsfile-libev.py

@@ -0,0 +1,9 @@
+import sys
+import toml
+
+pyproject = toml.load(sys.argv[1])
+base = pyproject["tool"]["cassandra-driver"]
+base["libev-includes"] = [sys.argv[2]]
+base["libev-libs"] = [sys.argv[3]]
+
+print(toml.dumps(pyproject))

pyproject.toml

@@ -51,6 +51,6 @@ include = ['cassandra', 'cassandra.io', 'cassandra.cqlengine', 'cassandra.graph'
 build-murmur3-extension = true
 build-libev-extension = true
 build-cython-extensions = true
-libev-includes = []
-libev-libs = []
+libev-includes = ["/usr/include/libev", "/usr/local/include", "/opt/local/include", "/usr/include"]
+libev-libs = ["/usr/local/lib", "/opt/local/lib", "/usr/lib64"]
 build-concurrency = 0

setup.py

@@ -30,6 +30,7 @@
 is_pypy = "PyPy" in sys.version
 is_supported_platform = sys.platform != "cli" and not sys.platform.startswith("java")
 is_supported_arch = sys.byteorder != "big"
+is_supported = is_supported_platform and is_supported_arch
 
 # ========================== A few upfront checks ==========================
 platform_unsupported_msg = \
@@ -66,62 +67,58 @@
     pyproject_data = toml.load(f)
 driver_project_data = pyproject_data["tool"]["cassandra-driver"]
 
-murmur3_ext = Extension('cassandra.cmurmur3', sources=['cassandra/cmurmur3.c'])
-
-DEFAULT_LIBEV_INCLUDES = ['/usr/include/libev', '/usr/local/include', '/opt/local/include', '/usr/include']
-DEFAULT_LIBEV_LIBS = ['/usr/local/lib', '/opt/local/lib', '/usr/lib64']
-libev_includes = driver_project_data["libev-includes"] or DEFAULT_LIBEV_INCLUDES
-libev_libs = driver_project_data["libev-libs"] or DEFAULT_LIBEV_LIBS
-if is_macos:
-    libev_includes.extend(['/opt/homebrew/include', os.path.expanduser('~/homebrew/include')])
-    libev_libs.extend(['/opt/homebrew/lib'])
-libev_ext = Extension('cassandra.io.libevwrapper',
-                      sources=['cassandra/io/libevwrapper.c'],
-                      include_dirs=libev_includes,
-                      libraries=['ev'],
-                      library_dirs=libev_libs)
-
-try_murmur3 = driver_project_data["build-murmur3-extension"] and is_supported_platform and is_supported_arch
-try_libev = driver_project_data["build-libev-extension"] and is_supported_platform and is_supported_arch
-try_cython = driver_project_data["build-cython-extensions"] and is_supported_platform and is_supported_arch and not is_pypy
+def key_or_false(k):
+    return driver_project_data[k] if k in driver_project_data else False
+
+try_murmur3 = key_or_false("build-murmur3-extension") and is_supported
+try_libev = key_or_false("build-libev-extension") and is_supported
+try_cython = key_or_false("build-cython-extensions") and is_supported and not is_pypy
 
 build_concurrency = driver_project_data["build-concurrency"]
 
-def build_extension_list():
-
-    rv = []
-
-    if try_murmur3:
-        sys.stderr.write("Appending murmur extension %s\n" % murmur3_ext)
-        rv.append(murmur3_ext)
-
-    if try_libev:
-        sys.stderr.write("Appending libev extension %s\n" % libev_ext)
-        rv.append(libev_ext)
-
-    if try_cython:
-        sys.stderr.write("Trying Cython builds in order to append Cython extensions\n")
-        try:
-            from Cython.Build import cythonize
-            cython_candidates = ['cluster', 'concurrent', 'connection', 'cqltypes', 'metadata',
-                                 'pool', 'protocol', 'query', 'util']
-            compile_args = [] if is_windows else ['-Wno-unused-function']
-            rv.extend(cythonize(
-                    [Extension('cassandra.%s' % m, ['cassandra/%s.py' % m],
-                                extra_compile_args=compile_args)
-                        for m in cython_candidates],
-                    nthreads=build_concurrency,
-                    exclude_failures=True))
-
-            rv.extend(cythonize(Extension("*", ["cassandra/*.pyx"], extra_compile_args=compile_args),
-                                          nthreads=build_concurrency))
-        except Exception as exc:
-            sys.stderr.write("Failed to cythonize one or more modules. These will not be compiled as extensions (optional).\n")
-            sys.stderr.write("Cython error: %s\n" % exc)
-    
-    return rv
+exts = []
+if try_murmur3:
+    murmur3_ext = Extension('cassandra.cmurmur3', sources=['cassandra/cmurmur3.c'])
+    sys.stderr.write("Appending murmur extension %s\n" % murmur3_ext)
+    exts.append(murmur3_ext)
+
+if try_libev:
+    libev_includes = driver_project_data["libev-includes"]
+    libev_libs = driver_project_data["libev-libs"]
+    if is_macos:
+        libev_includes.extend(['/opt/homebrew/include', os.path.expanduser('~/homebrew/include')])
+        libev_libs.extend(['/opt/homebrew/lib'])
+    libev_ext = Extension('cassandra.io.libevwrapper',
+                          sources=['cassandra/io/libevwrapper.c'],
+                          include_dirs=libev_includes,
+                          libraries=['ev'],
+                          library_dirs=libev_libs)
+    sys.stderr.write("Appending libev extension %s\n" % libev_ext)
+    exts.append(libev_ext)
+
+if try_cython:
+    sys.stderr.write("Trying Cython builds in order to append Cython extensions\n")
+    try:
+        from Cython.Build import cythonize
+        cython_candidates = ['cluster', 'concurrent', 'connection', 'cqltypes', 'metadata',
+                             'pool', 'protocol', 'query', 'util']
+        compile_args = [] if is_windows else ['-Wno-unused-function']
+        exts.extend(cythonize(
+            [Extension('cassandra.%s' % m, ['cassandra/%s.py' % m],
+                       extra_compile_args=compile_args)
+                       for m in cython_candidates],
+                       nthreads=build_concurrency,
+                       exclude_failures=True))
+
+        exts.extend(cythonize(
+            Extension("*", ["cassandra/*.pyx"],
+                      extra_compile_args=compile_args),
+                      nthreads=build_concurrency))
+    except Exception as exc:
+        sys.stderr.write("Failed to cythonize one or more modules. These will not be compiled as extensions (optional).\n")
+        sys.stderr.write("Cython error: %s\n" % exc)
 
 # ========================== And finally setup() itself ==========================
 setup(
-    ext_modules = build_extension_list()
+    ext_modules = exts
 )