update libicsneo & refactor build_libicsneo.py (#171)

Author: kschwarz-intrepidcs

.gitignore

@@ -102,4 +102,5 @@ msvc/ipch*
 
 # in-tree builds
 /libusb
-/libicsneo
+/libicsneo
+/libpcap

.readthedocs.yaml

@@ -8,11 +8,12 @@ version: 2
 build:
   os: ubuntu-22.04
   apt_packages:
+    - curl
+    - flex
+    - bison
     - clang
     - clang-format
     - build-essential
-    - libpcap0.8-dev 
-    - libusb-1.0-0-dev 
     - cmake
   tools:
     python: "3.11"

build_libicsneo.py

@@ -3,103 +3,143 @@
 import os
 import shutil
 import sys
+from pathlib import Path
 
-LIBICSNEO_COMMIT = "9ef93eb73e1ada72ccc7a571744b9db1a564f18b"
-LIBUSB_COMMIT = "4239bc3a50014b8e6a5a2a59df1fff3b7469543b"
+ROOT = Path.cwd()
 CPUS = str(multiprocessing.cpu_count())
 
+LIBUSB_VERSION = "1.0.27"
+LIBUSB_ROOT = f"{ROOT}/libusb/{LIBUSB_VERSION}"
+LIBUSB_SOURCE = f"{LIBUSB_ROOT}/source"
+LIBUSB_BUILD = f"{LIBUSB_ROOT}/build"
+LIBUSB_INSTALL = f"{LIBUSB_ROOT}/install"
 
-def checkout():
-    if not os.path.isdir("libicsneo"):
-        subprocess.check_output(["git", "clone", "https://github.com/intrepidcs/libicsneo.git"])
-    subprocess.check_output(["git", "checkout", LIBICSNEO_COMMIT], cwd="libicsneo")
+LIBICSNEO_VERSION = "e37d939"
+LIBICSNEO_ROOT = f"{ROOT}/libicsneo/{LIBICSNEO_VERSION}"
+LIBICSNEO_SOURCE = f"{LIBICSNEO_ROOT}/source"
+LIBICSNEO_BUILD = f"{LIBICSNEO_ROOT}/build"
+LIBICSNEO_INSTALL = f"{LIBICSNEO_ROOT}/install"
 
-    if not os.path.isdir("libusb") and sys.platform == "darwin":
-        subprocess.check_output(["git", "clone", "https://github.com/libusb/libusb.git"])
-        subprocess.check_output(["git", "checkout", LIBUSB_COMMIT], cwd="libusb")
+LIBPCAP_VERSION = "1.10.4"
+LIBPCAP_ROOT = f"{ROOT}/libpcap/{LIBPCAP_VERSION}"
+LIBPCAP_SOURCE = f"{LIBPCAP_ROOT}/source"
+LIBPCAP_BUILD = f"{LIBPCAP_ROOT}/build"
+LIBPCAP_INSTALL = f"{LIBPCAP_ROOT}/install"
 
+def checkout():
+    if not os.path.isdir(LIBICSNEO_SOURCE):
+        os.makedirs(LIBICSNEO_ROOT, exist_ok=True)
+        subprocess.check_output(["git", "clone", "https://github.com/intrepidcs/libicsneo.git", LIBICSNEO_SOURCE], cwd=LIBICSNEO_ROOT)
+        subprocess.check_output(["git", "checkout", LIBICSNEO_VERSION], cwd=LIBICSNEO_SOURCE)
+    
+    if not os.path.isdir(LIBUSB_SOURCE):
+        os.makedirs(LIBUSB_ROOT, exist_ok=True)
+        subprocess.check_output(["curl", "-LO", f"https://github.com/libusb/libusb/releases/download/v{LIBUSB_VERSION}/libusb-{LIBUSB_VERSION}.tar.bz2"], cwd=LIBUSB_ROOT)
+        subprocess.check_output(["tar", "-xf", f"libusb-{LIBUSB_VERSION}.tar.bz2"], cwd=LIBUSB_ROOT)
+        subprocess.check_output(["mv", f"libusb-{LIBUSB_VERSION}", "source"], cwd=LIBUSB_ROOT)
+        subprocess.check_output(["rm", f"libusb-{LIBUSB_VERSION}.tar.bz2"], cwd=LIBUSB_ROOT)
+
+    if not os.path.isdir(LIBPCAP_SOURCE):
+        os.makedirs(LIBPCAP_ROOT, exist_ok=True)
+        subprocess.check_output(["curl", "-LO", f"https://www.tcpdump.org/release/libpcap-{LIBPCAP_VERSION}.tar.gz"], cwd=LIBPCAP_ROOT)
+        subprocess.check_output(["tar", "-xf", f"libpcap-{LIBPCAP_VERSION}.tar.gz"], cwd=LIBPCAP_ROOT)
+        subprocess.check_output(["mv", f"libpcap-{LIBPCAP_VERSION}", "source"], cwd=LIBPCAP_ROOT)
+        subprocess.check_output(["rm", f"libpcap-{LIBPCAP_VERSION}.tar.gz"], cwd=LIBPCAP_ROOT)
 
 def _build_libusb():
-    subprocess.check_output(["git", "clean", "-xdf"], cwd="libusb")
-
-    subprocess.check_output(["./bootstrap.sh"], cwd="libusb")
-
+    os.makedirs(LIBUSB_BUILD, exist_ok=True)
     env = os.environ.copy()
-    env["CFLAGS"] = "-arch x86_64 -arch arm64 -mmacosx-version-min=10.13"
-    env["CXXFLAGS"] = "-arch x86_64 -arch arm64 -mmacosx-version-min=10.13"
-    subprocess.check_output(["./configure", "--disable-static", "--prefix=/tmp/libusb"], cwd="libusb", env=env)
-
-    subprocess.check_output(["make", "-j" + CPUS], cwd="libusb")
-    subprocess.check_output(["make", "install"], cwd="libusb")
-
-
-def _build_icsneo_linux():
+    if sys.platform == "darwin":
+        env["CFLAGS"] = "-arch x86_64 -arch arm64 -mmacosx-version-min=10.13"
+        env["CXXFLAGS"] = "-arch x86_64 -arch arm64 -mmacosx-version-min=10.13"
+    else:
+        env["CFLAGS"] = "-fPIC"
+        env["CXXFLAGS"] = "-fPIC"
+    subprocess.check_output([
+        f"{LIBUSB_SOURCE}/configure",
+        "--disable-shared",
+        "--enable-static",
+        f"--prefix={LIBUSB_INSTALL}",
+        "--disable-udev"
+    ], cwd=LIBUSB_BUILD, env=env)
+
+    subprocess.check_output(["make", "-j" + CPUS], cwd=LIBUSB_BUILD)
+    subprocess.check_output(["make", "install"], cwd=LIBUSB_BUILD)
+
+def _build_libpcap():
+    os.makedirs(LIBPCAP_BUILD, exist_ok=True)
+    env = os.environ.copy()
+    if sys.platform == "darwin":
+        env["CFLAGS"] = "-arch x86_64 -arch arm64 -mmacosx-version-min=10.13"
+        env["CXXFLAGS"] = "-arch x86_64 -arch arm64 -mmacosx-version-min=10.13"
+    else:
+        env["CFLAGS"] = "-fPIC"
+        env["CXXFLAGS"] = "-fPIC"
+    subprocess.check_output([
+        f"{LIBPCAP_SOURCE}/configure",
+        "--disable-shared",
+        f"--prefix={LIBPCAP_INSTALL}",
+    ], cwd=LIBPCAP_BUILD, env=env)
+
+    subprocess.check_output(["make", "-j" + CPUS], cwd=LIBPCAP_BUILD)
+    subprocess.check_output(["make", "install"], cwd=LIBPCAP_BUILD)
+
+def _build_libicsneo_linux():
     subprocess.check_output(["git", "clean", "-xdf"], cwd="libicsneo")
     subprocess.check_output(["mkdir", "-p", "libicsneo/build"])
 
     subprocess.check_output(
-        ["cmake", "..", "-DCMAKE_BUILD_TYPE=Release", "-DLIBICSNEO_BUILD_ICSNEOLEGACY=ON"], cwd="libicsneo/build"
+        [
+            "cmake",
+            "-DCMAKE_BUILD_TYPE=Release",
+            "-DLIBICSNEO_BUILD_ICSNEOLEGACY=ON",
+            f"-DCMAKE_PREFIX_PATH={LIBUSB_INSTALL};{LIBPCAP_INSTALL}",
+            "-S", LIBICSNEO_SOURCE,
+            "-B", LIBICSNEO_BUILD
+        ]
     )
     subprocess.check_output(
-        ["cmake", "--build", ".", "--target", "icsneolegacy", "--parallel", CPUS], cwd="libicsneo/build"
+        ["cmake", "--build", LIBICSNEO_BUILD, "--target", "icsneolegacy", "--parallel", CPUS]
     )
 
-
-def _build_icsneo_macos():
-    subprocess.check_output(["git", "clean", "-xdf"], cwd="libicsneo")
-    subprocess.check_output(["mkdir", "-p", "libicsneo/build"])
-
+def _build_libicsneo_macos():
     subprocess.check_output(
         [
             "cmake",
-            "..",
             "-DCMAKE_BUILD_TYPE=Release",
             "-DCMAKE_OSX_ARCHITECTURES=arm64;x86_64",
             "-DLIBICSNEO_BUILD_ICSNEOLEGACY=ON",
             "-DCMAKE_OSX_DEPLOYMENT_TARGET=10.13",
-            "-DLIBUSB_LIBRARIES=/tmp/libusb/lib/libusb-1.0.0.dylib",
-            "-DLIBUSB_INCLUDE_DIR=/tmp/libusb/include/libusb-1.0",
-        ],
-        cwd="libicsneo/build",
+            f"-DCMAKE_PREFIX_PATH={LIBUSB_INSTALL};{LIBPCAP_INSTALL}",
+            "-S", LIBICSNEO_SOURCE,
+            "-B", LIBICSNEO_BUILD
+        ]
     )
 
     subprocess.check_output(
-        ["cmake", "--build", ".", "--target", "icsneolegacy", "--parallel", CPUS], cwd="libicsneo/build"
-    )
-    subprocess.check_output(
-        [
-            "install_name_tool",
-            "-change",
-            "/tmp/libusb/lib/libusb-1.0.0.dylib",
-            "@loader_path/libusb-1.0.0.dylib",
-            "libicsneolegacy.dylib",
-        ],
-        cwd="libicsneo/build",
+        ["cmake", "--build", LIBICSNEO_BUILD, "--target", "icsneolegacy", "--parallel", CPUS]
     )
 
-
 def build():
     print("Building libicsneo...")
+    _build_libusb()
+    _build_libpcap()
     if sys.platform == "darwin":
-        _build_libusb()
-        _build_icsneo_macos()
+        _build_libicsneo_macos()
     elif sys.platform == "linux":
-        _build_icsneo_linux()
-
+        _build_libicsneo_linux()
 
 def copy():
     if sys.platform == "darwin":
-        shutil.copyfile("/tmp/libusb/lib/libusb-1.0.0.dylib", "ics/libusb-1.0.0.dylib")
-        shutil.copyfile("libicsneo/build/libicsneolegacy.dylib", "ics/libicsneolegacy.dylib")
+        shutil.copyfile(f"{LIBICSNEO_BUILD}/libicsneolegacy.dylib", "ics/libicsneolegacy.dylib")
     elif sys.platform == "linux":
-        shutil.copyfile("libicsneo/build/libicsneolegacy.so", "ics/libicsneolegacy.so")
+        shutil.copyfile(f"{LIBICSNEO_BUILD}/libicsneolegacy.so", "ics/libicsneolegacy.so")
 
 
 def clean():
     shutil.rmtree("libicsneo")
     shutil.rmtree("libusb")
-    shutil.rmtree("/tmp/libusb")
-
+    shutil.rmtree("libpcap")
 
 if __name__ == "__main__":
     if "--clean" in sys.argv:

build_mac.sh

@@ -3,11 +3,11 @@
 python_directory="/Library/Frameworks/Python.framework/Versions"
 python_versions=("3.7" "3.8" "3.9" "3.10" "3.11")
 
-# libusb dependencies
-brew install automake libtool cmake clang-format
+# dependencies
+brew install cmake clang-format
 ret_value=$?
 if [ $ret_value -ne 0 ]; then
-    echo "Failed to install automake/libtoolgit..."
+    echo "Failed to install clang/clang-format..."
     exit $ret_value
 fi
 

doc/source/header.rst

@@ -173,12 +173,12 @@ python_ics does not provide binaries for linux distributions so we will have to
 Fedora Dependencies (FC28)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-``sudo dnf install redhat-rpm-config gcc g++ python3-devel clang clang-tools-extra``
+``sudo dnf install redhat-rpm-config gcc g++ python3-devel clang clang-tools-extra flex bison``
 
 Debian/Ubuntu Dependencies
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-``sudo apt install build-essential python-dev clang-format``
+``sudo apt install build-essential python-dev clang-format flex bison``
 
 libicsneo library
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

pyproject.toml

@@ -6,11 +6,11 @@ before-build = "python -m pip install wheel"
 manylinux-x86_64-image = "quay.io/pypa/manylinux_2_28_x86_64"
 
 [tool.cibuildwheel.linux]
-before-build = "dnf install -y libpcap-devel cmake libusbx-devel clang clang-tools-extra"
+before-build = "dnf install -y cmake clang clang-tools-extra flex bison"
 
 [tool.cibuildwheel.macos]
 archs = ["universal2"]
-before-build = "brew install automake libtool cmake clang-format"
+before-build = "brew install cmake clang-format"
 
 [tool.cibuildwheel.windows]
 before-build = "choco install llvm"

travis/build-wheels.sh

@@ -12,7 +12,7 @@ function repair_wheel {
 
 
 # Install a system package required by our library
-dnf install -y clang clang-tools-extra
+dnf install -y clang clang-tools-extra flex bison
 
 # Compile wheels
 for PYBIN in /opt/python/*/bin; do