Skip to content

Commit 5a731c9

Browse files
hzqmwnehzqmwne
authored
[INLONG-12090][SDK] Build standard Python Dataproxy SDK wheels based on PEP-517 (#12091)
1 parent 560c1ec commit 5a731c9

6 files changed

Lines changed: 207 additions & 1 deletion

File tree

‎inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-docker/Dockerfile_python‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
FROM quay.io/pypa/manylinux2014_x86_64:latest
21+
22+
ENV CMAKE_POLICY_VERSION_MINIMUM=3.5
23+
24+
# build cpp sdk
25+
26+
COPY dataproxy-sdk-cpp /workspace/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp
27+
WORKDIR /workspace/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp
28+
29+
RUN ./build_third_party.sh
30+
RUN ./build.sh
31+
32+
# prepare python sdk source tree
33+
34+
COPY dataproxy-sdk-python /workspace/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python
35+
WORKDIR /workspace/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python
36+
37+
RUN cp -r ../dataproxy-sdk-cpp ./
38+
39+
# build python sdk wheels using PEP-517
40+
41+
RUN /opt/python/cp38-cp38/bin/python3 -m build --sdist
42+
43+
RUN /opt/python/cp38-cp38/bin/python3 -m pip wheel --no-deps --wheel-dir dist dist/*.tar.gz
44+
RUN /opt/python/cp39-cp39/bin/python3 -m pip wheel --no-deps --wheel-dir dist dist/*.tar.gz
45+
RUN /opt/python/cp310-cp310/bin/python3 -m pip wheel --no-deps --wheel-dir dist dist/*.tar.gz
46+
RUN /opt/python/cp311-cp311/bin/python3 -m pip wheel --no-deps --wheel-dir dist dist/*.tar.gz
47+
RUN /opt/python/cp312-cp312/bin/python3 -m pip wheel --no-deps --wheel-dir dist dist/*.tar.gz
48+
RUN /opt/python/cp313-cp313/bin/python3 -m pip wheel --no-deps --wheel-dir dist dist/*.tar.gz
49+
RUN /opt/python/cp314-cp314/bin/python3 -m pip wheel --no-deps --wheel-dir dist dist/*.tar.gz
50+
51+
RUN auditwheel repair dist/*.whl
52+
53+
# now target sdist is in dist/, and target wheels are in wheelhouse/. ignore wheels in dist/.

‎inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-docker/README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,3 +62,10 @@ docker build -t inlong/dataproxy-sdk-build:latest .
6262
```bash
6363
docker build --build-arg PYTHON_VERSION=3.9.18 -t inlong/dataproxy-sdk-build:py39 .
6464
```
65+
66+
### Build Python SDK wheels
67+
68+
```bash
69+
cd inlong-sdk/dataproxy-sdk-twins
70+
docker build -f dataproxy-sdk-docker/Dockerfile_python .
71+
```

‎inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/CMakeLists.txt‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,13 @@ include_directories("${CMAKE_CURRENT_SOURCE_DIR}/dataproxy-sdk-cpp/third_party/l
3030
include_directories("${CMAKE_CURRENT_SOURCE_DIR}/dataproxy-sdk-cpp/third_party/lib64")
3131
include_directories("${CMAKE_CURRENT_SOURCE_DIR}/dataproxy-sdk-cpp/release/lib")
3232

33-
add_subdirectory(pybind11)
33+
# Prefer local pybind11/ if present for legacy build; otherwise use pybind11 from PyPI via PEP-517.
34+
if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/pybind11/CMakeLists.txt")
35+
add_subdirectory(pybind11)
36+
else ()
37+
set(PYBIND11_FINDPYTHON ON)
38+
find_package(pybind11 CONFIG REQUIRED)
39+
endif ()
3440

3541
link_directories("${CMAKE_CURRENT_SOURCE_DIR}/dataproxy-sdk-cpp/third_party/lib")
3642
link_directories("${CMAKE_CURRENT_SOURCE_DIR}/dataproxy-sdk-cpp/third_party/lib64")
@@ -39,3 +45,10 @@ link_directories("${CMAKE_CURRENT_SOURCE_DIR}/dataproxy-sdk-cpp/release/lib")
3945
pybind11_add_module(inlong_dataproxy inlong_dataproxy.cpp)
4046

4147
target_link_libraries(inlong_dataproxy PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/dataproxy-sdk-cpp/release/lib/dataproxy_sdk.a" liblog4cplusS.a libsnappy.a libcurl.a libssl.a libcrypto.a)
48+
49+
# For scikit-build wheels: install into the Python platlib directory, include target so and other required files like pyi.
50+
# It is good that cmake removes rpath in so during install.
51+
if (DEFINED SKBUILD_PLATLIB_DIR)
52+
install(TARGETS inlong_dataproxy LIBRARY DESTINATION "${SKBUILD_PLATLIB_DIR}")
53+
install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/inlong_dataproxy.pyi" DESTINATION "${SKBUILD_PLATLIB_DIR}")
54+
endif ()

‎inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/README.md‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,23 @@ After the build process finished, you can import the package (`import inlong_dat
8686

8787
> **Note**: When the C++ SDK or the version of Python you're using is updated, you'll need to rebuild it using either of the above methods (Native Build or Docker Build).
8888
89+
### Method 3: PEP 517 Build
90+
91+
1. Build the C++ SDK. see [dataproxy-sdk-cpp/README.md](../dataproxy-sdk-cpp/README.md)
92+
93+
2. Copy `dataproxy-sdk-cpp` dir into `dataproxy-sdk-python`
94+
```bash
95+
cp -r ../dataproxy-sdk-cpp ./
96+
```
97+
98+
3. Go to the `dataproxy-sdk-python` directory, build the Python SDK
99+
```bash
100+
python3 -m pip install build
101+
python3 -m build
102+
```
103+
104+
Also, you can refer [dataproxy-sdk-docker/README.md](../dataproxy-sdk-docker/README.md) to build python sdk wheels for manylinux.
105+
89106
## Config Parameters
90107

91108
Refer to `demo/config_example.json`.

‎inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/inlong_dataproxy.pyi‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
from collections.abc import Callable
21+
from typing import Optional
22+
23+
UserCallBack = Callable[[str, str, str, int, int, str], int]
24+
# e.g. def callback_func(inlong_group_id: str, inlong_stream_id: str, msg: str, msg_len: int, report_time: int, ip: str) -> int:
25+
26+
class InLongApi:
27+
def __init__(self) -> None: ...
28+
def init_api(self, config_path: str) -> int: ...
29+
def send(
30+
self,
31+
groupId: str,
32+
streamId: str,
33+
msg: str,
34+
msgLen: int,
35+
pyCallback: Optional[UserCallBack] = ..., # None means no callback
36+
) -> int: ...
37+
def close_api(self, timeout_ms: int) -> int: ...

‎inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/pyproject.toml‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
[build-system]
21+
requires = [
22+
"scikit-build-core>=0.12", "pybind11",
23+
]
24+
build-backend = "scikit_build_core.build"
25+
26+
[project]
27+
name = "inlong-dataproxy-sdk"
28+
version = "2.3.0"
29+
description = "Python bindings for Apache InLong DataProxy SDK"
30+
readme = "README.md"
31+
requires-python = ">=3.8"
32+
license = { text = "Apache-2.0" }
33+
authors = [
34+
{ name = "Apache InLong" },
35+
]
36+
classifiers = [
37+
"License :: OSI Approved :: Apache Software License",
38+
"Programming Language :: Python :: 3",
39+
"Programming Language :: C++",
40+
"Operating System :: POSIX :: Linux",
41+
]
42+
43+
[project.urls]
44+
Homepage = "https://github.com/apache/inlong"
45+
Source = "https://github.com/apache/inlong/tree/master/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python"
46+
47+
[tool.scikit-build]
48+
minimum-version = "0.12"
49+
50+
# Do NOT run CMake when building sdist.
51+
sdist.cmake = false
52+
53+
# Make sdist minimal via a strict allowlist.
54+
55+
sdist.inclusion-mode = "manual"
56+
57+
# First exclude all files
58+
sdist.exclude = [
59+
"**/*",
60+
]
61+
62+
# Then only include necessary files
63+
# NOTICE: sdist.include always takes precedence over sdist.exclude
64+
sdist.include = [
65+
# --- Python binding sources/build config ---
66+
"/pyproject.toml",
67+
"/CMakeLists.txt",
68+
"/inlong_dataproxy.cpp",
69+
"/inlong_dataproxy.pyi",
70+
"/README.md",
71+
72+
# Headers referenced by current CMake include_directories
73+
"/dataproxy-sdk-cpp/src/core/*.h",
74+
75+
# Static libraries used by linking
76+
"/dataproxy-sdk-cpp/release/lib/*.a",
77+
"/dataproxy-sdk-cpp/third_party/lib/*.a",
78+
"/dataproxy-sdk-cpp/third_party/lib64/*.a",
79+
]

0 commit comments

Comments
 (0)