Merge changes I87f6584a,I0d8e22bb
* changes:
Switch to generating pip packages in a Debian container
Add --local_test mode for mirror_pip_packages.py
diff --git a/tools/python/generate_pip_packages.Dockerfile b/tools/python/generate_pip_packages.Dockerfile
new file mode 100644
index 0000000..dcdd840
--- /dev/null
+++ b/tools/python/generate_pip_packages.Dockerfile
@@ -0,0 +1,30 @@
+FROM debian:bullseye
+
+RUN apt-get update
+RUN apt-get install -y \
+ curl \
+ clang-13
+
+# Get latest patchelf for auditwheel.
+RUN curl -L https://github.com/NixOS/patchelf/releases/download/0.15.0/patchelf-0.15.0-x86_64.tar.gz > /tmp/patchelf.tar.gz \
+ && tar -xaf /tmp/patchelf.tar.gz -C /usr \
+ && rm -f /tmp/patchelf.tar.gz
+
+# Get the same Python that we're using for for actually running Python code.
+RUN mkdir /opt/python/
+RUN curl -SL \
+ https://github.com/indygreg/python-build-standalone/releases/download/20220802/cpython-3.9.13+20220802-x86_64-unknown-linux-gnu-install_only.tar.gz \
+ | tar -xz -C /opt/
+
+# Install dependencies of the pip packages that we're compiling.
+RUN apt-get install -y \
+ libcairo2-dev \
+ libgirepository1.0-dev \
+ libglib2.0-0 \
+ libgtk-3-dev
+
+# Make some symlinks to satisfy assumptions some of the installer scripts (e.g.
+# setup.py files) make about the system.
+RUN ln -s /opt/python/ /install
+RUN ln -s /usr/bin/clang-13 /usr/bin/clang && \
+ ln -s /usr/bin/clang++-13 /usr/bin/clang++
diff --git a/tools/python/generate_pip_packages_in_docker.sh b/tools/python/generate_pip_packages_in_docker.sh
index 77cd068..a3af2ae 100755
--- a/tools/python/generate_pip_packages_in_docker.sh
+++ b/tools/python/generate_pip_packages_in_docker.sh
@@ -20,10 +20,9 @@
readonly PLAT="$1"
readonly ARCH="$2"
-readonly PYTHON_VERSION="$3"
-readonly CALLER_ID="$4"
+readonly CALLER_ID="$3"
-readonly PYTHON_BIN="/opt/python/cp${PYTHON_VERSION}-cp${PYTHON_VERSION}/bin/python3"
+readonly PYTHON_BIN="/opt/python/bin/python3"
# Try to make the wheels reproducible by telling them we're in 1980.
# Unfortunately, this is insufficient due to a pip bug.
@@ -51,7 +50,7 @@
source venv/bin/activate
-readonly -a PIP_BIN=("${PYTHON_BIN}" -m pip)
+readonly -a PIP_BIN=(pip)
# Might be useful for debugging.
"${PIP_BIN[@]}" --version
@@ -60,6 +59,7 @@
# Get wheels for everything. Everything is stored in a temporary wheelhouse in
# case we need to run the "auditwheel" tool against them.
+"${PIP_BIN[@]}" install wheel
"${PIP_BIN[@]}" wheel \
--no-deps \
-r "${SCRIPT_DIR}/requirements.lock.txt" \
@@ -98,6 +98,7 @@
# libraries into the wheel itself. The list of system libraries that will not
# get grafted is here:
# https://peps.python.org/pep-0599/#the-manylinux2014-policy
+"${PIP_BIN[@]}" install auditwheel
for wheel in "${wheels_built_from_source[@]}"; do
wheel_path="${SCRIPT_DIR}/wheelhouse_tmp/${wheel}"
echo "Repairing wheel ${wheel}"
diff --git a/tools/python/mirror_pip_packages.py b/tools/python/mirror_pip_packages.py
index 2b18406..19bb477 100644
--- a/tools/python/mirror_pip_packages.py
+++ b/tools/python/mirror_pip_packages.py
@@ -20,8 +20,7 @@
import requests
from pkginfo import Wheel
-PYTHON_VERSION = 39
-PLAT = "manylinux_2_28"
+PLAT = "manylinux_2_31"
ARCH = "x86_64"
WHEELHOUSE_MIRROR_URL = "https://software.frc971.org/Build-Dependencies/wheelhouse"
PY_DEPS_WWWW_DIR = "/var/www/html/files/frc971/Build-Dependencies/wheelhouse"
@@ -124,6 +123,13 @@
"extreme caution! This may easily cause issues with building "
"older commits. Use this only if you know what you're doing."))
parser.add_argument(
+ "-l",
+ "--local_test",
+ action="store_true",
+ help=("If set, generate the URL overrides pointing at the generated "
+ "local files. Incompatible with --ssh_host. This is useful for "
+ "iterating on generated wheel files."))
+ parser.add_argument(
"--ssh_host",
type=str,
help=("The SSH host to copy the downloaded Go repositories to. This "
@@ -138,6 +144,18 @@
python_dir = root_dir / "tools" / "python"
+ container_tag = f"pip-compile:{caller}"
+
+ subprocess.run([
+ "docker",
+ "build",
+ "--file=generate_pip_packages.Dockerfile",
+ f"--tag={container_tag}",
+ ".",
+ ],
+ cwd=python_dir,
+ check=True)
+
# Run the wheel generation script inside the docker container provided by
# the pypa/manylinux project.
# https://github.com/pypa/manylinux/
@@ -147,11 +165,10 @@
"-it",
"-v",
f"{python_dir}:/opt/971_build/",
- f"quay.io/pypa/{PLAT}_{ARCH}",
+ container_tag,
"/opt/971_build/generate_pip_packages_in_docker.sh",
PLAT,
ARCH,
- str(PYTHON_VERSION),
str(caller_id),
],
check=True)
@@ -166,6 +183,10 @@
override_information = {}
for wheel in sorted(wheels):
wheel_url = f"{WHEELHOUSE_MIRROR_URL}/{wheel.name}"
+ if args.local_test:
+ override_url = f"file://{wheel.resolve()}"
+ else:
+ override_url = wheel_url
sha256 = compute_file_sha256(wheel)
# Check if we already have the wheel uploaded. If so, download that one
@@ -175,10 +196,13 @@
wheel_found, sha256_on_mirror = search_for_uploaded_wheel(
wheel, wheel_url)
+ if args.local_test:
+ wheel_found = False
+
if args.force:
if wheel_found and sha256 != sha256_on_mirror:
print(
- f"WARNING: The next upload wheel change sha256 for {wheel}!"
+ f"WARNING: The next upload will change sha256 for {wheel}!"
)
wheels_to_be_uploaded.append(wheel)
else:
@@ -192,7 +216,7 @@
# requirements.lock.txt file uses.
info = Wheel(wheel)
override_information[f"{info.name.lower()}=={info.version}"] = {
- "url": wheel_url,
+ "url": override_url,
"sha256": sha256,
}