Upgrade amd64 sysroot to bookworm
Check in the script too used to generate the rootfs. This makes it
easier to make changes going forwards and to add more to the rootfs.
Change-Id: I36e5a1272e6bcb8a8848b6cd2f35befc45f57273
Signed-off-by: Austin Schuh <austin.linux@gmail.com>
diff --git a/WORKSPACE b/WORKSPACE
index 46a10c5..efeb2a4 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -324,7 +324,7 @@
"linux-aarch64": llvm_opt_copts,
},
standard_libraries = {
- "linux-x86_64": "libstdc++-10",
+ "linux-x86_64": "libstdc++-12",
"linux-aarch64": "libstdc++-12",
},
static_libstdcxx = False,
@@ -459,17 +459,12 @@
url = "https://software.frc971.org/Build-Dependencies/2023-11-18-bookworm-arm64-nvidia-rootfs.tar.xz",
)
-# Created with:
-# `debootstrap buster buster_sysroot`
-# and then chrooting in and running:
-# apt install libc6-dev libstdc++-7-dev
-# removing the apt cache,
-# and then tarring up the result
+# Sysroot generated using //frc971/amd64/build_rootfs.py
http_archive(
name = "amd64_debian_sysroot",
- build_file = "@//:compilers/debian_rootfs.BUILD",
- sha256 = "5e10f4cac85a98a39da1716b218bc05fff4666c61cc471a7df27876710bc86d2",
- url = "https://software.frc971.org/Build-Dependencies/2022-01-06-debian-bullseye_rootfs.tar.bz2",
+ build_file = "@//:compilers/amd64_debian_rootfs.BUILD",
+ sha256 = "3c098330f8bc57dccb4191167cfbba4c47f3bacf52926479c95ad2e50834b3c2",
+ url = "https://software.frc971.org/Build-Dependencies/2023-11-09-bookworm-amd64-nvidia-rootfs.tar.xz",
)
local_repository(
diff --git a/aos/ipc_lib/lockless_queue_test.cc b/aos/ipc_lib/lockless_queue_test.cc
index 14701e2..5b57aa9 100644
--- a/aos/ipc_lib/lockless_queue_test.cc
+++ b/aos/ipc_lib/lockless_queue_test.cc
@@ -461,7 +461,7 @@
LocklessQueueReader::Result read_result = reader.Read(
i, &monotonic_sent_time, &realtime_sent_time, &monotonic_remote_time,
&realtime_remote_time, &remote_queue_index, &source_boot_uuid, &length,
- &(read_data[0]), std::ref(should_read_callback));
+ &(read_data[0]), should_read_callback);
if (read_result != LocklessQueueReader::Result::GOOD) {
if (read_result == LocklessQueueReader::Result::TOO_OLD) {
diff --git a/aos/ipc_lib/queue_racer.cc b/aos/ipc_lib/queue_racer.cc
index 0b8f1a6..27f3835 100644
--- a/aos/ipc_lib/queue_racer.cc
+++ b/aos/ipc_lib/queue_racer.cc
@@ -320,11 +320,16 @@
const uint32_t wrapped_i =
i % static_cast<size_t>(QueueIndex::MaxIndex(
0xffffffffu, LocklessQueueSize(queue_.memory())));
- LocklessQueueReader::Result read_result = reader.Read(
- wrapped_i, &monotonic_sent_time, &realtime_sent_time,
- &monotonic_remote_time, &realtime_remote_time, &remote_queue_index,
- &source_boot_uuid, &length, &(read_data[0]),
- set_should_read ? std::ref(should_read) : std::ref(nop));
+ LocklessQueueReader::Result read_result =
+ set_should_read
+ ? reader.Read(wrapped_i, &monotonic_sent_time, &realtime_sent_time,
+ &monotonic_remote_time, &realtime_remote_time,
+ &remote_queue_index, &source_boot_uuid, &length,
+ &(read_data[0]), std::ref(should_read))
+ : reader.Read(wrapped_i, &monotonic_sent_time, &realtime_sent_time,
+ &monotonic_remote_time, &realtime_remote_time,
+ &remote_queue_index, &source_boot_uuid, &length,
+ &(read_data[0]), nop);
// The code in lockless_queue.cc reads everything but data, checks that the
// header hasn't changed, then reads the data. So, if we succeed and both
diff --git a/compilers/amd64_debian_rootfs.BUILD b/compilers/amd64_debian_rootfs.BUILD
new file mode 100644
index 0000000..8bec407
--- /dev/null
+++ b/compilers/amd64_debian_rootfs.BUILD
@@ -0,0 +1,48 @@
+filegroup(
+ name = "sysroot_files",
+ srcs = glob(
+ # TODO(austin): Only include the base files here. Need to figure out what those are.
+ # TODO(austin): Generate that list when building the rootfs?
+ include = [
+ "include/**",
+ "lib/**",
+ "lib64/**",
+ "usr/include/**",
+ "usr/lib/**",
+ "usr/bin/**",
+ "usr/lib64/**",
+ ],
+ exclude = [
+ "usr/share/**",
+ "usr/include/thrust/**",
+ "usr/include/nv/**",
+ "usr/include/cuda/**",
+ "usr/include/cub/**",
+ "usr/bin/X11",
+ ],
+ ),
+ visibility = ["//visibility:public"],
+)
+
+cc_library(
+ name = "nppi",
+ srcs = [
+ "usr/lib/x86_64-linux-gnu/libnppc.so.11",
+ "usr/lib/x86_64-linux-gnu/libnppif.so.11",
+ ],
+ hdrs = glob(
+ include = ["usr/include/nppi*.h"],
+ ),
+ visibility = ["//visibility:public"],
+)
+
+cc_library(
+ name = "cudart",
+ srcs = [
+ "usr/lib/x86_64-linux-gnu/libcuda.so.1",
+ "usr/lib/x86_64-linux-gnu/libcudart.so.11.0",
+ ],
+ visibility = ["//visibility:public"],
+)
+
+# TODO(austin): lzma, gstreamer, opencv
diff --git a/compilers/debian_rootfs.BUILD b/compilers/debian_rootfs.BUILD
deleted file mode 100644
index 7eef9ec..0000000
--- a/compilers/debian_rootfs.BUILD
+++ /dev/null
@@ -1,17 +0,0 @@
-filegroup(
- name = "sysroot_files",
- srcs = glob(
- include = [
- "include/**",
- "lib/**",
- "lib64/**",
- "usr/include/**",
- "usr/lib/**",
- "usr/lib64/**",
- ],
- exclude = [
- "usr/share/**",
- ],
- ),
- visibility = ["//visibility:public"],
-)
diff --git a/frc971/amd64/build_rootfs.py b/frc971/amd64/build_rootfs.py
new file mode 100755
index 0000000..cc0138e
--- /dev/null
+++ b/frc971/amd64/build_rootfs.py
@@ -0,0 +1,181 @@
+#!/usr/bin/python3
+
+import contextlib
+import datetime
+import pathlib
+import subprocess
+import shlex
+import os
+import sys
+
+REQUIRED_DEPS = ["debootstrap"]
+
+ROOTFS_FOLDER = "/tmp/rootfs"
+
+
+@contextlib.contextmanager
+def scoped_bind_mount(partition):
+ """Bind mounts a folder from the host into the rootfs."""
+ result = subprocess.run(
+ ["sudo", "mount", "--bind", partition, f"{ROOTFS_FOLDER}/{partition}"],
+ check=True)
+
+ try:
+ yield partition
+ finally:
+ subprocess.run(["sudo", "umount", f"{ROOTFS_FOLDER}/{partition}"],
+ check=True)
+
+
+def check_required_deps(deps):
+ """Checks if the provided list of dependencies is installed."""
+ missing_deps = []
+ for dep in deps:
+ result = subprocess.run(["dpkg-query", "-W", "-f='${Status}'", dep],
+ check=True,
+ stdout=subprocess.PIPE)
+
+ if "install ok installed" not in result.stdout.decode('utf-8'):
+ missing_deps.append(dep)
+
+ if len(missing_deps) > 0:
+ print("Missing dependencies, please install:")
+ print("sudo apt-get install", " ".join(missing_deps))
+ return True
+
+ return False
+
+
+def target_unescaped(cmd):
+ """Runs a command as root with bash -c cmd, ie without escaping."""
+ subprocess.run([
+ "sudo", "chroot", "--userspec=0:0", f"{ROOTFS_FOLDER}", "/bin/bash",
+ "-c", cmd
+ ],
+ check=True)
+
+
+def target(cmd):
+ """Runs a command as root with escaping."""
+ target_unescaped(shlex.join([shlex.quote(c) for c in cmd]))
+
+
+def copyfile(owner, permissions, file):
+ """Copies a file from contents/{file} with the provided owner and permissions."""
+ print("copyfile", owner, permissions, file)
+ subprocess.run(
+ ["sudo", "cp", f"contents/{file}", f"{ROOTFS_FOLDER}/{file}"],
+ check=True)
+ subprocess.run(["sudo", "chmod", permissions, f"{ROOTFS_FOLDER}/{file}"],
+ check=True)
+ target(["chown", owner, f"/{file}"])
+
+
+def target_symlink(owner, permissions, link_target, linkname):
+ full_linkname = f"{ROOTFS_FOLDER}/{linkname}"
+ print(link_target)
+ print(full_linkname)
+ if not os.path.exists(full_linkname):
+ target(["ln", "-s", link_target, linkname])
+
+ assert (pathlib.Path(full_linkname).is_symlink())
+
+ target(["chown", owner, linkname])
+ target(["chmod", permissions, linkname])
+
+
+def target_mkdir(owner_group, permissions, folder):
+ """Creates a directory recursively with the provided permissions and ownership."""
+ print("target_mkdir", owner_group, permissions, folder)
+ owner, group = owner_group.split('.')
+ target(
+ ["install", "-d", "-m", permissions, "-o", owner, "-g", group, folder])
+
+
+def main():
+ if check_required_deps(REQUIRED_DEPS):
+ return 1
+
+ new_image = not os.path.exists(ROOTFS_FOLDER)
+ if new_image:
+ os.mkdir(ROOTFS_FOLDER)
+
+ if new_image:
+ subprocess.run([
+ "sudo", "debootstrap", "--no-check-gpg", "bookworm", ROOTFS_FOLDER,
+ "http://deb.debian.org/debian/"
+ ],
+ check=True)
+
+ if not os.path.exists(
+ f"{ROOTFS_FOLDER}/etc/apt/sources.list.d/bullseye-backports.list"):
+ copyfile("root.root", "644",
+ "etc/apt/sources.list.d/bullseye-backports.list")
+ target(["apt-get", "update"])
+
+ with scoped_bind_mount("/dev") as _:
+ with scoped_bind_mount("/proc") as _:
+ target([
+ "apt-get",
+ "-y",
+ "install",
+ "libopencv-calib3d406",
+ "libopencv-contrib406",
+ "libopencv-core406",
+ "libopencv-features2d406",
+ "libopencv-flann406",
+ "libopencv-highgui406",
+ "libopencv-imgcodecs406",
+ "libopencv-imgproc406",
+ "libopencv-ml406",
+ "libopencv-objdetect406",
+ "libopencv-photo406",
+ "libopencv-shape406",
+ "libopencv-stitching406",
+ "libopencv-superres406",
+ "libopencv-video406",
+ "libopencv-videoio406",
+ "libopencv-videostab406",
+ "libopencv-viz406",
+ "libv4l-dev",
+ "libc6-dev",
+ "libstdc++-12-dev",
+ "nvidia-cuda-dev",
+ "nvidia-cuda-toolkit",
+ ])
+
+ target_mkdir("root.root", "755", "usr/lib/cuda/bin")
+ target_symlink("root.root", "555", "../../../bin/fatbinary",
+ "usr/lib/cuda/bin/x86_64-unknown-linux-gnu-fatbinary")
+
+ target(["apt-get", "clean"])
+
+ target(["ldconfig"])
+
+ tarball = datetime.date.today().strftime(
+ f"{os.getcwd()}/%Y-%m-%d-bookworm-amd64-nvidia-rootfs.tar")
+ print(tarball)
+
+ subprocess.run([
+ "sudo",
+ "tar",
+ "--exclude=./usr/share/ca-certificates",
+ "--exclude=./usr/src",
+ "--exclude=./usr/lib/mesa-diverted",
+ "--exclude=./usr/bin/X11",
+ "--exclude=./usr/lib/systemd/system/system-systemd*cryptsetup.slice",
+ "--exclude=./dev",
+ "-cf",
+ tarball,
+ ".",
+ ],
+ cwd=ROOTFS_FOLDER,
+ check=True)
+
+ subprocess.run(["sha256sum", tarball], check=True)
+
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/frc971/amd64/contents/etc/apt/sources.list.d/bullseye-backports.list b/frc971/amd64/contents/etc/apt/sources.list.d/bullseye-backports.list
new file mode 100644
index 0000000..219e4bc
--- /dev/null
+++ b/frc971/amd64/contents/etc/apt/sources.list.d/bullseye-backports.list
@@ -0,0 +1,12 @@
+deb http://deb.debian.org/debian/ bookworm main contrib non-free non-free-firmware
+deb-src http://deb.debian.org/debian/ bookworm main contrib non-free non-free-firmware
+
+deb https://security.debian.org/debian-security bookworm-security main contrib non-free non-free-firmware
+deb-src https://security.debian.org/debian-security bookworm-security main contrib non-free non-free-firmware
+
+# bookworm-updates, previously known as 'volatile'
+deb http://deb.debian.org/debian/ bookworm-updates main contrib non-free non-free-firmware
+deb-src http://deb.debian.org/debian/ bookworm-updates main contrib non-free non-free-firmware
+
+deb http://deb.debian.org/debian bullseye-backports main
+deb http://deb.debian.org/debian bullseye main
diff --git a/frc971/halide_generator.sh b/frc971/halide_generator.sh
index 01cb29c..d6e5d41 100755
--- a/frc971/halide_generator.sh
+++ b/frc971/halide_generator.sh
@@ -33,8 +33,8 @@
-fcolor-diagnostics \
-I"${HALIDE}/include" \
-nostdinc \
- -isystem"${SYSROOT}/usr/include/c++/10" \
- -isystem"${SYSROOT}/usr/include/${MULTIARCH}/c++/10" \
+ -isystem"${SYSROOT}/usr/include/c++/12" \
+ -isystem"${SYSROOT}/usr/include/${MULTIARCH}/c++/12" \
-isystem"${SYSROOT}/usr/include/c++/7/backward" \
-isystem"${LLVM_TOOLCHAIN}/lib/clang/17/include" \
-isystem"${SYSROOT}/usr/include/${MULTIARCH}" \
diff --git a/y2020/vision/sift/fast_gaussian_halide_generator.sh b/y2020/vision/sift/fast_gaussian_halide_generator.sh
index 560e54f..87fb20d 100755
--- a/y2020/vision/sift/fast_gaussian_halide_generator.sh
+++ b/y2020/vision/sift/fast_gaussian_halide_generator.sh
@@ -40,8 +40,8 @@
-fcolor-diagnostics \
-I"${HALIDE}/include" \
-nostdinc \
- -isystem"${SYSROOT}/usr/include/c++/10" \
- -isystem"${SYSROOT}/usr/include/${MULTIARCH}/c++/10" \
+ -isystem"${SYSROOT}/usr/include/c++/12" \
+ -isystem"${SYSROOT}/usr/include/${MULTIARCH}/c++/12" \
-isystem"${SYSROOT}/usr/include/c++/7/backward" \
-isystem"${LLVM_TOOLCHAIN}/lib/clang/17/include" \
-isystem"${SYSROOT}/usr/include/${MULTIARCH}" \