Skip to content

Commit 0725d1e

Browse files
tuhaihetuhaihe
committed
CI: migrate from Jenkins to GitHub Actions
* Move CI/CD pipeline from Jenkins to GitHub Actions * Split workflows into two separate files: - license-check.yml: Apache RAT and license compliance checks - build-test.yml: MADlib build and test process * Preserve existing PostgreSQL configurations and test setup * Maintain compatibility with current test exclusion rules * Keep tool scripts in original locations for reference * Rename Jenkinsfile to Jenkinsfile.deprecated This migration enables: - Better integration with GitHub ecosystem - Improved visibility of CI/CD processes - Parallel execution of license checks and build tests - Native support for pull request workflows Add your commit body here
1 parent b195300 commit 0725d1e

3 files changed

Lines changed: 228 additions & 0 deletions

File tree

‎.github/workflows/build-test.yml‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: Build and Test
19+
20+
on:
21+
push:
22+
branches: [ madlib2-master ]
23+
pull_request:
24+
branches: [ madlib2-master ]
25+
# Allow manual triggers for testing
26+
workflow_dispatch:
27+
28+
jobs:
29+
build-and-test:
30+
name: Build and Test MADlib
31+
runs-on: ubuntu-latest
32+
33+
services:
34+
postgres:
35+
image: madlib/postgres_15:jenkins
36+
env:
37+
POSTGRES_PASSWORD: postgres
38+
ports:
39+
- 5432:5432
40+
options: >-
41+
--health-cmd pg_isready
42+
--health-interval 10s
43+
--health-timeout 5s
44+
--health-retries 5
45+
--ulimit core=-1
46+
--privileged
47+
48+
steps:
49+
- uses: actions/checkout@v4
50+
51+
- name: Set up Python
52+
uses: actions/setup-python@v4
53+
with:
54+
python-version: '3.x'
55+
56+
- name: Install Python dependencies
57+
run: |
58+
python -m pip install --upgrade pip
59+
pip install mock pandas numpy xgboost scikit-learn pyyaml pyxb-x pypmml
60+
61+
- name: Set up build environment
62+
run: |
63+
sudo apt-get update
64+
sudo apt-get install -y \
65+
build-essential \
66+
cmake \
67+
openjdk-11-jre-headless \
68+
postgresql-15
69+
70+
# Configure PostgreSQL
71+
sudo cp tool/pg_hba.conf.postgres /etc/postgresql/15/main/pg_hba.conf
72+
echo "* soft nproc unlimited" | sudo tee /etc/security/limits.d/postgres-limits.conf
73+
sudo service postgresql start
74+
75+
- name: Build MADlib
76+
run: |
77+
mkdir build
78+
cd build
79+
cmake ..
80+
make -j$(nproc)
81+
sudo make install
82+
make package
83+
84+
- name: Run Tests
85+
run: |
86+
cd build
87+
export PATH=$PATH:/usr/lib/postgresql/15/bin/
88+
89+
# Install MADlib
90+
src/bin/madpack -s mad -p postgres -c postgres/postgres@localhost:5432/postgres install
91+
92+
mkdir -p /tmp
93+
94+
# Remove known problematic test files (as done in jenkins_build.sh)
95+
rm -rf src/ports/postgres/modules/deep_learning/test
96+
rm -rf src/ports/postgres/15/modules/deep_learning/test
97+
rm -rf src/ports/postgres/modules/linalg/test/linalg.sql_in
98+
rm -rf src/ports/postgres/modules/prob/test/prob.sql_in
99+
rm -rf src/ports/postgres/modules/stats/test/cox_prop_hazards.sql_in
100+
rm -rf src/ports/postgres/modules/utilities/test/path.sql_in
101+
rm -rf src/ports/postgres/modules/mxgboost/test/madlib_xgboost.sql_in
102+
rm -rf src/ports/postgres/modules/kmeans/test/kmeans.sql_in
103+
104+
# Run tests
105+
src/bin/madpack -s mad -p postgres -c postgres/postgres@localhost:5432/postgres -d /tmp dev-check
106+
src/bin/madpack -s mad -p postgres -c postgres/postgres@localhost:5432/postgres -d /tmp unit-test
107+
108+
- name: Process Test Results
109+
run: |
110+
python3 tool/jenkins/junit_export.py $PWD $PWD/logs/madlib_dev_check.log $PWD/logs/madlib_dev_check.xml
111+
112+
- name: Publish Test Results
113+
uses: EnricoMi/publish-unit-test-result-action@v2
114+
if: always()
115+
with:
116+
files: "logs/madlib_dev_check.xml"
117+
118+
- name: Upload Build Artifacts
119+
uses: actions/upload-artifact@v3
120+
with:
121+
name: madlib-packages
122+
path: build/*.deb

‎.github/workflows/license-check.yml‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: License Check
19+
20+
on:
21+
push:
22+
branches: [ madlib2-master ]
23+
pull_request:
24+
branches: [ madlib2-master ]
25+
# Allow manual triggers for compliance verification
26+
workflow_dispatch:
27+
28+
jobs:
29+
rat-check:
30+
name: Apache RAT License Check
31+
runs-on: ubuntu-latest
32+
33+
steps:
34+
- uses: actions/checkout@v4
35+
36+
- name: Set up JDK
37+
uses: actions/setup-java@v3
38+
with:
39+
java-version: '11'
40+
distribution: 'temurin'
41+
42+
- name: Set up Maven
43+
uses: actions/setup-java@v3
44+
with:
45+
java-version: '11'
46+
distribution: 'temurin'
47+
cache: 'maven'
48+
49+
- name: Run Apache RAT Check
50+
id: rat
51+
run: |
52+
# Run RAT check and capture output
53+
{
54+
echo "### Apache RAT Check Results :mag:" >> $GITHUB_STEP_SUMMARY
55+
echo "Running Maven RAT check..." >> $GITHUB_STEP_SUMMARY
56+
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
57+
mvn apache-rat:check 2>&1 | tee rat_output.txt
58+
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
59+
60+
# Add summary of unapproved licenses if any
61+
if grep -q "Unapproved Licenses" rat_output.txt; then
62+
echo "#### ❌ Found Unapproved Licenses" >> $GITHUB_STEP_SUMMARY
63+
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
64+
grep -A 10 "Unapproved Licenses" rat_output.txt >> $GITHUB_STEP_SUMMARY
65+
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
66+
else
67+
echo "#### ✅ No Unapproved Licenses Found" >> $GITHUB_STEP_SUMMARY
68+
fi
69+
} || {
70+
echo "#### ❌ RAT Check Failed" >> $GITHUB_STEP_SUMMARY
71+
exit 1
72+
}
73+
74+
- name: Run Additional License Checks
75+
id: additional
76+
run: |
77+
{
78+
echo "### Additional License Checks :clipboard:" >> $GITHUB_STEP_SUMMARY
79+
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
80+
81+
chmod +x ./tool/jenkins/rat_check.sh
82+
./tool/jenkins/rat_check.sh 2>&1 | tee license_checks.txt
83+
84+
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
85+
86+
# Add summary based on exit code
87+
if [ $? -eq 0 ]; then
88+
echo "#### Results:" >> $GITHUB_STEP_SUMMARY
89+
echo "- ✅ NOTICE file year is current" >> $GITHUB_STEP_SUMMARY
90+
echo "- ✅ Version numbers match" >> $GITHUB_STEP_SUMMARY
91+
echo "- ✅ No unexpected binary files found" >> $GITHUB_STEP_SUMMARY
92+
else
93+
echo "#### ❌ Additional Checks Failed" >> $GITHUB_STEP_SUMMARY
94+
echo "See log output above for details." >> $GITHUB_STEP_SUMMARY
95+
fi
96+
}
97+
98+
- name: Upload RAT Report
99+
if: always()
100+
uses: actions/upload-artifact@v3
101+
with:
102+
name: rat-report
103+
path: |
104+
target/rat.txt
105+
rat_output.txt
106+
license_checks.txt

Jenkinsfile renamed to Jenkinsfile.deprecated

File renamed without changes.

0 commit comments

Comments
 (0)