Skip to content

Commit b8c4d3c

Browse files
vsochvsoch
committed
refactor: test pixi
It would be nice to have a consistent interface for Python. Signed-off-by: vsoch <vsoch@users.noreply.github.com>
1 parent 9c36063 commit b8c4d3c

5 files changed

Lines changed: 109 additions & 92 deletions

File tree

‎.github/scripts/build-bindings.sh‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
3+
here=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
4+
FLUX_REPO=${1:-0}
5+
FLUX_BRANCH=${2:-3.11}
6+
7+
echo "Flux Repo: ${FLUX_REPO}"
8+
echo "Flux Branch: ${FLUX_BRANCH}"
9+
10+
# Use default values if inputs are empty
11+
FLUX_REPO_URL=${FLUX_REPO:-"https://github.com/flux-framework/flux-core"}
12+
FLUX_BRANCH_NAME=${FLUX_BRANCH:-"master"}
13+
14+
git clone -b ${FLUX_BRANCH_NAME} ${FLUX_REPO_URL} /tmp/flux-core
15+
mv /tmp/flux-core/src/bindings/python/flux ./flux
16+
python setup.py sdist
17+
18+
# I think ldconfig is Linux-specific
19+
sudo ldconfig || echo "Not running on Linux"

‎.github/scripts/setup.sh‎

Lines changed: 35 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,12 @@
33
# Exit on error, unset variable, or pipe failure
44
set -euo pipefail
55

6-
# This will be empty for nightly test, and we will clone master branch
7-
FLUX_RELEASE_VERSION=${FLUX_RELEASE_VERSION:-0.78.0}
8-
FLUX_VERSION=${FLUX_VERSION:-0.78.0}
6+
# This must be set
7+
PYTHON_VERSION=${1:-py311}
8+
9+
# These will be empty for nightly test, and we will clone master branch
10+
FLUX_RELEASE_VERSION=${2:-0.78.0}
11+
FLUX_VERSION=${3:-0.78.0}
912

1013
if [[ "$(uname)" == "Darwin" ]]; then
1114
brew install \
@@ -18,35 +21,26 @@ if [[ "$(uname)" == "Darwin" ]]; then
1821
libev \
1922
zeromq \
2023
jansson \
21-
lz4 \
2224
libarchive \
23-
hwloc \
24-
sqlite \
25-
lua \
26-
luarocks \
27-
cffi \
28-
libyaml \
29-
jq
25+
hwloc
3026

3127
# Possibility - install munge with MacPorts
3228
# sudo port install munge
3329
eval "$(/opt/homebrew/bin/brew shellenv)"
3430

3531
# Set the dynamic library path variable for macOS
3632
export DYLD_LIBRARY_PATH=/usr/local/lib
37-
33+
3834
# Ensure we activate micromamabe that has a pinned version
39-
PIP_INSTALL="/opt/conda/bin/python3 -m pip install"
4035
ln -s $(which glibtoolize) /usr/local/bin/libtoolize
4136

4237
CPPFLAGS="-I${HOMEBREW_PREFIX}/include/lua"
4338
CPPFLAGS="-I$(brew --prefix libev)/include ${CPPFLAGS}"
4439
CPPFLAGS="-I$(brew --prefix epoll-shim)/include/libepoll-shim ${CPPFLAGS}"
45-
# LDFLAGS=-L${HOMEBREW_PREFIX}/lib
40+
4641
PKG_CONFIG_PATH=$(pkg-config --variable pc_path pkg-config)
4742
PKG_CONFIG_PATH=$(brew --prefix libarchive)/lib/pkgconfig:${PKG_CONFIG_PATH}
4843
PATH=$(brew --prefix libtool)/libexec/gnubin:$PATH
49-
export LD_LIBRARY_PATH="/opt/conda/envs/build/lib:/opt/conda/envs/build/lib64"
5044

5145
# Linux block!
5246
else
@@ -76,8 +70,6 @@ else
7670
libs3-dev \
7771
libevent-dev \
7872
libarchive-dev \
79-
python3-sphinx \
80-
python3-cffi \
8173
libtool \
8274
git \
8375
build-essential \
@@ -86,53 +78,49 @@ else
8678
sudo rm -rf /var/lib/apt/lists/*
8779
sudo ldconfig
8880

89-
# Set the dynamic library path variable for Linux. The conda path is for the container.
90-
export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/opt/conda/envs/build/lib
81+
# Set the dynamic library path variable for Linux.
82+
export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib
83+
echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"
9184
fi
9285

93-
# Ensure we find micromamba environment first
94-
# mamba init commands are intolerable so let's do basics ourselves.
95-
PIP_INSTALL="/opt/conda/envs/build/bin/python3 -m pip install"
96-
export PATH="/opt/conda/envs/build/bin:$PATH"
97-
export PYTHONPATH=$(find /opt/conda/envs/build -name site-packages)
98-
export PYTHON=/opt/conda/envs/build/bin/python3
99-
export PYTHON_PREFIX=${PYTHONPATH}
100-
export PYTHON_EXEC_PREFIX=${PYTHONPATH}
101-
# export LIBS="-L/opt/conda/envs/build/lib:/opt/conda/envs/build/lib64"
102-
10386
echo "Found Python $(which python3)"
104-
echo "PYTHONPATH: ${PATH}"
10587
echo "PATH: ${PATH}"
106-
echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"
10788

10889
# Here we can start common setup (we hope)
10990
echo "Flux Version for pypi is ${FLUX_VERSION}"
11091
if [[ "$(uname)" == "Darwin" ]]; then
92+
PIXI_ROOT=/Users/runner/work/flux-python/flux-python/.pixi
11193
sed -i '' "s/package_version = \"develop\"/package_version = \"$FLUX_VERSION\"/" setup.py
94+
echo $(find /Users/runner/ -name python3) || echo $(find /Users/runner/ -name python)
95+
export PYTHON=${PIXI_ROOT}/envs/${PYTHON_VERSION}/bin/python3
96+
# Tell where to look for lua.h
97+
CPPFLAGS="-I${PIXI_ROOT}/envs/${PYTHON_VERSION}/include ${CPPFLAGS}"
98+
export PYTHONPATH=$(find ${PIXI_ROOT}/envs/${PYTHON_VERSION}/lib/ -name site-packages)
11299
else
100+
PIXI_ROOT=/home/runner/work/flux-python/flux-python/.pixi
113101
sed -i "s/package_version = \"develop\"/package_version = \"$FLUX_VERSION\"/" setup.py
102+
CPPFLAGS="-I${PIXI_ROOT}/envs/${PYTHON_VERSION}/include"
114103
fi
115104

105+
# Can we remove need for setuptools?
106+
export PYTHON=${PIXI_ROOT}/envs/${PYTHON_VERSION}/bin/python3
107+
export PYTHON_NOVERSIONCHECK=yes
108+
export LUA_CFLAGS="-I${PIXI_ROOT}/envs/${PYTHON_VERSION}/include"
109+
export LDFLAGS="-L$PIXI_ROOT/envs/${PYTHON_VERSION}/lib -L$PIXI_ROOT/envs/${PYTHON_VERSION}/lib64"
110+
111+
echo "Found lua.h in $(find ${PIXI_ROOT} -name lua.h)" || echo "Did not find lua.h"
112+
116113
# Store current directory
117114
here=$(pwd)
115+
luarocks install luaposix
118116

119-
# Install Python requirements using the OS-specific pip command - allow for ubuntu to be 24.04 or not.
120-
# I like how "--break-system-packages" is analogous with --i-acknowledge-i-am-a-terrible-person
121-
if [[ "$(uname)" == "Darwin" ]]; then
122-
echo "Installing python requirements for MacOS"
123-
${PIP_INSTALL} IPython setuptools ply sphinx cffi pyyaml
124-
${PIP_INSTALL} -r .github/scripts/requirements-dev.txt
125-
else
126-
echo "Installing python requirements for Linux"
127-
${PIP_INSTALL} IPython pyyaml || ${PIP_INSTALL} IPython --break-system-packages pyyaml
128-
${PIP_INSTALL} -r .github/scripts/requirements-dev.txt || ${PIP_INSTALL} -r .github/scripts/requirements-dev.txt --break-system-packages
129-
130-
# Flux Security --
131-
# Does not have a variant for Mac
117+
# Flux Security --
118+
# Does not have a variant for Mac
119+
if [[ "$(uname)" != "Darwin" ]]; then
132120
git clone https://github.com/flux-framework/flux-security ~/security
133121
cd ~/security
134122
./autogen.sh
135-
PYTHON_PREFIX=PYTHON_EXEC_PREFIX=${PYTHON_PREFIX} PYTHON=/opt/conda/bin/python3 ./configure --prefix=/usr/local
123+
./configure --prefix=/usr/local
136124
make
137125
sudo make install
138126
sudo ldconfig
@@ -149,11 +137,11 @@ chmod +x etc/gen-cmdhelp.py
149137
./autogen.sh || echo "No autogen here"
150138

151139
if [[ "$(uname)" == "Darwin" ]]; then
152-
CPPFLAGS="$CPPFLAGS" PKG_CONFIG_PATH=$PKG_CONFIG_PATH ./configure --prefix=/usr/local
140+
CPPFLAGS="$CPPFLAGS" CFLAGS="${CPPFLAGS}" LDFLAGS="${LDFLAGS}" PKG_CONFIG_PATH=$PKG_CONFIG_PATH ./configure --prefix=/usr/local
153141
ls ./scripts
154142
./scripts/check-macos.sh || echo "No check-macos.sh"
155143
else
156-
./configure --prefix=/usr/local
144+
./configure CPPFLAGS="$CPPFLAGS" CFLAGS="${CPPFLAGS}" LDFLAGS="${LDFLAGS}" --prefix=/usr/local
157145
make VERBOSE=1
158146
sudo make install || true
159147
sudo make install

‎.github/workflows/build-release.yaml‎

Lines changed: 7 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -28,52 +28,22 @@ jobs:
2828
matrix:
2929
# The matrix now includes both os and python
3030
os: [ubuntu-latest, macos-latest]
31-
python: ["3.8", "3.9", "3.10", "3.11"]
31+
python: ["py38", "py39", "py310", "py311", "py312"]
3232

3333
steps:
3434
- uses: actions/checkout@v4
35-
- name: Setup Python
36-
env:
37-
python_version: ${{ matrix.python }}
38-
run: /bin/bash ./docker/install-mamba.sh ${{ matrix.python }}
35+
- uses: prefix-dev/setup-pixi@v0.9.0
36+
with:
37+
environments: ${{ matrix.python }}
3938

40-
# The login shell ensures the mamba env is active
4139
- name: Build Flux Core Branch
42-
env:
43-
FLUX_RELEASE_VERSION: ${{ inputs.release_version }}
44-
FLUX_VERSION: ${{ inputs.version }}
45-
run: /bin/bash .github/scripts/setup.sh
40+
run: pixi run -e ${{ matrix.python }} bash .github/scripts/setup.sh ${{ matrix.python }} ${{ inputs.release_version }} ${{ inputs.version }}
4641

4742
- name: Build Python Bindings
48-
shell: bash -l {0}
49-
env:
50-
FLUX_BRANCH: ${{ inputs.branch }}
51-
FLUX_REPO: ${{ inputs.repo }}
52-
run: |
53-
echo "Flux Repo: ${{ inputs.repo }}"
54-
echo "Flux Branch: ${{ inputs.branch }}"
55-
56-
# Use default values if inputs are empty
57-
FLUX_REPO_URL=${FLUX_REPO:-"https://github.com/flux-framework/flux-core"}
58-
FLUX_BRANCH_NAME=${FLUX_BRANCH:-"master"}
59-
60-
git clone -b ${FLUX_BRANCH_NAME} ${FLUX_REPO_URL} /tmp/flux-core
61-
mv /tmp/flux-core/src/bindings/python/flux ./flux
62-
python setup.py sdist
63-
64-
# I think ldconfig is Linux-specific
65-
if [[ "${{ runner.os }}" == "Linux" ]]; then
66-
sudo ldconfig
67-
fi
43+
run: pixi run -e ${{ matrix.python }} bash .github/scripts/build-bindings.sh ${{ inputs.branch }} ${{ inputs.repo }}
6844

6945
- name: Build Python Wheels
70-
shell: bash -l {0}
71-
env:
72-
build_number: ${{ inputs.rc }}
73-
python_version: ${{ matrix.python }}
74-
run: |
75-
/bin/bash ./docker/build-wheels.sh ${{ env.build_number }} ${{ env.python_version }}
76-
ls ./dist
46+
run: pixi run -e ${{ matrix.python }} bash ./docker/build-wheels.sh ${{ env.build_number }} ${{ env.python_version }}
7747

7848
- name: Upload distributions
7949
uses: actions/upload-artifact@v4

‎docker/build-wheels.sh‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,7 @@ version=${2:-3.11}
77
# This is intended to run in the container
88
echo "Building Python version ${version}"
99

10-
export PATH=/opt/conda/envs/build/bin:$PATH
11-
export PYTHONPATH=/opt/conda/envs/build/lib/python${version}/site-packages
12-
13-
/opt/conda/bin/mamba activate build || true
14-
1510
# Build the bindings for this python version!
16-
/opt/conda/envs/build/bin/python3 setup.py sdist
17-
/opt/conda/envs/build/bin/python3 setup.py bdist_wheel --plat-name=any --build-number=${build_number}
18-
unset PYTHONPATH
11+
python3 setup.py sdist
12+
python3 setup.py bdist_wheel --plat-name=any --build-number=${build_number}
13+
ls ./dist

‎pixi.toml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
[workspace]
2+
name = "flux-python"
3+
channels = ["conda-forge"]
4+
platforms = ["linux-64", "osx-64"]
5+
6+
[dependencies]
7+
python = ">=3.8"
8+
pip = "*"
9+
10+
[feature.py38.dependencies]
11+
python = "3.8.*"
12+
[feature.py39.dependencies]
13+
python = "3.9.*"
14+
[feature.py310.dependencies]
15+
python = "3.10.*"
16+
[feature.py311.dependencies]
17+
python = "3.11.*"
18+
[feature.py312.dependencies]
19+
python = "3.12.*"
20+
21+
[feature.common.dependencies]
22+
cffi = "*"
23+
pyyaml = "*"
24+
jsonschema = "*"
25+
docutils = "*"
26+
ply = "*"
27+
black = "*"
28+
ipython = "*"
29+
wheel = "*"
30+
setuptools = "*"
31+
luarocks = "*"
32+
lua = "*"
33+
lua-lpeg = "*"
34+
sphinx = "*"
35+
36+
[pypi-dependencies]
37+
setuptools = "*"
38+
39+
[environments]
40+
py38 = { features = ["py38", "common"] }
41+
py39 = { features = ["py39", "common"] }
42+
py310 = { features = ["py310", "common"] }
43+
py311 = { features = ["py311", "common"] }
44+
py312 = { features = ["py312", "common"] }
45+

0 commit comments

Comments
 (0)