Switch to generating pip packages in a Debian container
This patch switches us to use the upstream Debian Bullseye image for
building pip packages.
I couldn't find any specific reason to do this, but I was worried
about the upstream AlmaLinux-based one causing subtle issues.
Signed-off-by: Philipp Schrader <philipp.schrader@gmail.com>
Change-Id: I87f6584a3d7b4657d689947fdd16835b147f18b7
diff --git a/tools/python/generate_pip_packages.Dockerfile b/tools/python/generate_pip_packages.Dockerfile
new file mode 100644
index 0000000..dcdd840
--- /dev/null
+++ b/tools/python/generate_pip_packages.Dockerfile
@@ -0,0 +1,30 @@
+FROM debian:bullseye
+
+RUN apt-get update
+RUN apt-get install -y \
+ curl \
+ clang-13
+
+# Get latest patchelf for auditwheel.
+RUN curl -L https://github.com/NixOS/patchelf/releases/download/0.15.0/patchelf-0.15.0-x86_64.tar.gz > /tmp/patchelf.tar.gz \
+ && tar -xaf /tmp/patchelf.tar.gz -C /usr \
+ && rm -f /tmp/patchelf.tar.gz
+
+# Get the same Python that we're using for for actually running Python code.
+RUN mkdir /opt/python/
+RUN curl -SL \
+ https://github.com/indygreg/python-build-standalone/releases/download/20220802/cpython-3.9.13+20220802-x86_64-unknown-linux-gnu-install_only.tar.gz \
+ | tar -xz -C /opt/
+
+# Install dependencies of the pip packages that we're compiling.
+RUN apt-get install -y \
+ libcairo2-dev \
+ libgirepository1.0-dev \
+ libglib2.0-0 \
+ libgtk-3-dev
+
+# Make some symlinks to satisfy assumptions some of the installer scripts (e.g.
+# setup.py files) make about the system.
+RUN ln -s /opt/python/ /install
+RUN ln -s /usr/bin/clang-13 /usr/bin/clang && \
+ ln -s /usr/bin/clang++-13 /usr/bin/clang++
diff --git a/tools/python/generate_pip_packages_in_docker.sh b/tools/python/generate_pip_packages_in_docker.sh
index 77cd068..a3af2ae 100755
--- a/tools/python/generate_pip_packages_in_docker.sh
+++ b/tools/python/generate_pip_packages_in_docker.sh
@@ -20,10 +20,9 @@
readonly PLAT="$1"
readonly ARCH="$2"
-readonly PYTHON_VERSION="$3"
-readonly CALLER_ID="$4"
+readonly CALLER_ID="$3"
-readonly PYTHON_BIN="/opt/python/cp${PYTHON_VERSION}-cp${PYTHON_VERSION}/bin/python3"
+readonly PYTHON_BIN="/opt/python/bin/python3"
# Try to make the wheels reproducible by telling them we're in 1980.
# Unfortunately, this is insufficient due to a pip bug.
@@ -51,7 +50,7 @@
source venv/bin/activate
-readonly -a PIP_BIN=("${PYTHON_BIN}" -m pip)
+readonly -a PIP_BIN=(pip)
# Might be useful for debugging.
"${PIP_BIN[@]}" --version
@@ -60,6 +59,7 @@
# Get wheels for everything. Everything is stored in a temporary wheelhouse in
# case we need to run the "auditwheel" tool against them.
+"${PIP_BIN[@]}" install wheel
"${PIP_BIN[@]}" wheel \
--no-deps \
-r "${SCRIPT_DIR}/requirements.lock.txt" \
@@ -98,6 +98,7 @@
# libraries into the wheel itself. The list of system libraries that will not
# get grafted is here:
# https://peps.python.org/pep-0599/#the-manylinux2014-policy
+"${PIP_BIN[@]}" install auditwheel
for wheel in "${wheels_built_from_source[@]}"; do
wheel_path="${SCRIPT_DIR}/wheelhouse_tmp/${wheel}"
echo "Repairing wheel ${wheel}"
diff --git a/tools/python/mirror_pip_packages.py b/tools/python/mirror_pip_packages.py
index aa8ed88..19bb477 100644
--- a/tools/python/mirror_pip_packages.py
+++ b/tools/python/mirror_pip_packages.py
@@ -20,8 +20,7 @@
import requests
from pkginfo import Wheel
-PYTHON_VERSION = 39
-PLAT = "manylinux_2_28"
+PLAT = "manylinux_2_31"
ARCH = "x86_64"
WHEELHOUSE_MIRROR_URL = "https://software.frc971.org/Build-Dependencies/wheelhouse"
PY_DEPS_WWWW_DIR = "/var/www/html/files/frc971/Build-Dependencies/wheelhouse"
@@ -145,6 +144,18 @@
python_dir = root_dir / "tools" / "python"
+ container_tag = f"pip-compile:{caller}"
+
+ subprocess.run([
+ "docker",
+ "build",
+ "--file=generate_pip_packages.Dockerfile",
+ f"--tag={container_tag}",
+ ".",
+ ],
+ cwd=python_dir,
+ check=True)
+
# Run the wheel generation script inside the docker container provided by
# the pypa/manylinux project.
# https://github.com/pypa/manylinux/
@@ -154,11 +165,10 @@
"-it",
"-v",
f"{python_dir}:/opt/971_build/",
- f"quay.io/pypa/{PLAT}_{ARCH}",
+ container_tag,
"/opt/971_build/generate_pip_packages_in_docker.sh",
PLAT,
ARCH,
- str(PYTHON_VERSION),
str(caller_id),
],
check=True)