Support downloading packages for other architectures
This is really handy for working with the armhf and arm64 systems we
build code for.
Signed-off-by: Brian Silverman <bsilver16384@gmail.com>
Change-Id: I966df8da77d6a985c4e3f5b1ad8b2b637bed3f98
diff --git a/debian/download_packages.py b/debian/download_packages.py
index e3bb845..e8caebd 100755
--- a/debian/download_packages.py
+++ b/debian/download_packages.py
@@ -2,23 +2,54 @@
import sys
import os
+import os.path
import re
import subprocess
import tempfile
+import urllib.request
import argparse
import hashlib
-def get_deps(package):
+def initialize_apt(apt_dir, apt_args, args):
+ os.mkdir(os.path.join(apt_dir, 'etc'))
+ os.mkdir(os.path.join(apt_dir, 'etc', 'apt'))
+ os.mkdir(os.path.join(apt_dir, 'etc', 'apt', 'trusted.gpg.d'))
+ os.mkdir(os.path.join(apt_dir, 'etc', 'apt', 'preferences.d'))
+ os.mkdir(os.path.join(apt_dir, 'var'))
+ os.mkdir(os.path.join(apt_dir, 'var', 'lib'))
+ os.mkdir(os.path.join(apt_dir, 'var', 'lib', 'dpkg'))
+ with open(os.path.join(apt_dir, 'var', 'lib', 'dpkg', 'status'), 'w'):
+ pass
+ with open(os.path.join(apt_dir, 'etc', 'apt', 'sources.list'), 'w') as f:
+ f.write("""
+deb http://deb.debian.org/debian/ {release} main contrib non-free
+deb-src http://deb.debian.org/debian/ {release} main contrib non-free
+
+deb https://security.debian.org/debian-security {release}-security main contrib non-free
+deb-src https://security.debian.org/debian-security {release}-security main contrib non-free
+
+deb http://deb.debian.org/debian/ {release}-updates main contrib non-free
+deb-src http://deb.debian.org/debian/ {release}-updates main contrib non-free
+
+deb http://deb.debian.org/debian {release}-backports main contrib non-free
+deb-src http://deb.debian.org/debian {release}-backports main contrib non-free
+""".format(release=args.release))
+ for key in args.apt_key:
+ basename = os.path.basename(key)
+ urllib.request.urlretrieve(key, os.path.join(apt_dir, 'etc', 'apt', 'trusted.gpg.d', basename))
+ subprocess.check_call(["apt-get"] + apt_args + ["update"])
+
+def get_deps(apt_args, package):
env = dict(os.environ)
del env['LD_LIBRARY_PATH']
- out = subprocess.check_output(["apt-rdepends", package], env=env)
+ out = subprocess.check_output(["apt-rdepends"] + apt_args + [package], env=env)
deps = out.splitlines()
return set([dep for dep in deps if not dep.startswith(b" ")])
-def get_all_deps(packages):
+def get_all_deps(apt_args, packages):
deps = set()
for package in packages or ():
- deps.update(get_deps(package))
+ deps.update(get_deps(apt_args, package))
return deps
def map_virtual_packages(packages):
@@ -63,15 +94,15 @@
continue
yield package
-def download_deps(packages, excludes, force_includes):
- deps = get_all_deps(packages)
- exclude_deps = get_all_deps(excludes)
+def download_deps(apt_args, packages, excludes, force_includes):
+ deps = get_all_deps(apt_args, packages)
+ exclude_deps = get_all_deps(apt_args, excludes)
deps -= exclude_deps
- force_include_deps = get_all_deps(force_includes)
+ force_include_deps = get_all_deps(apt_args, force_includes)
deps |= force_include_deps
env = dict(os.environ)
del env['LD_LIBRARY_PATH']
- subprocess.check_call([b"apt-get", b"download"] + list(map_virtual_packages(deps)), env=env)
+ subprocess.check_call([b"apt-get"] + [a.encode('utf-8') for a in apt_args] + [b"download"] + list(map_virtual_packages(deps)), env=env)
def fixup_files():
# Gotta remove those pesky epoch numbers in the file names. Bazel doesn't
@@ -114,15 +145,34 @@
parser = argparse.ArgumentParser()
parser.add_argument("--exclude", "-e", type=str, action="append", help="A package to exclude from the list")
parser.add_argument("--force-include", type=str, action="append", help="Force include this and its dependencies. Even if listed in excludes.")
+ parser.add_argument("--arch", type=str, default="amd64", help="Architecture to download files for.")
+ parser.add_argument("--apt-dir", type=str, help=" ".join([
+ "File to generate and store apt files in.",
+ "Helpful for saving time when downloading multiple groups of packages.",
+ "Some flags will be ignored in favor of the values used to create this folder, so be careful.",
+ ]))
+ parser.add_argument("--release", type=str, default="bullseye", help="Debian release to use.")
+ parser.add_argument("--apt-key", type=str, action="append", default=[
+ "https://ftp-master.debian.org/keys/archive-key-11.asc",
+ "https://ftp-master.debian.org/keys/archive-key-11-security.asc",
+ ], help="URL of an additional apt archive key to trust.")
parser.add_argument("package", nargs="+", help="The packages to download.")
args = parser.parse_args(argv[1:])
+ if args.apt_dir:
+ apt_dir = args.apt_dir
+ else:
+ apt_dir = tempfile.mkdtemp()
+ apt_args = ["-o", "Dir=" + apt_dir, "-o", "APT::Architecture=" + args.arch]
+ if not args.apt_dir:
+ print("Creating apt files in %s" % apt_dir)
+ initialize_apt(apt_dir, apt_args, args)
folder = tempfile.mkdtemp()
os.chdir(folder)
excludes = args.exclude or []
# Exclude common packages that don't make sense to include in everything all
# the time.
excludes += _ALWAYS_EXCLUDE
- download_deps(args.package, excludes, args.force_include)
+ download_deps(apt_args, args.package, excludes, args.force_include)
fixup_files()
print_file_list()
print("Your packages are all in %s" % folder)
diff --git a/debian/packages.bzl b/debian/packages.bzl
index 2914388..0f2784a 100644
--- a/debian/packages.bzl
+++ b/debian/packages.bzl
@@ -5,9 +5,11 @@
#
# 1. Create a "download_packages" build step in //debian/BUILD. List the
# packages you care about and exclude the ones you don't care about.
-# Invoke "bazel build" on the "download_packages" target you just created.
+# Invoke "bazel run" on the "download_packages" target you just created.
# Save the "_files" dictionary it prints into a .bzl file in the //debian
# folder. You will need to have the apt-rdepends package installed.
+# If you want to get packages for a different architecture or distribution,
+# you can pass flags here to control those. See the --help for details.
# 2. The "download_packages" steps prints the location of the deb packages
# after it prints the "_files" dictionary. Take the deb packages from there
# and upload them to https://www.frc971.org/Build-Dependencies/.
@@ -22,8 +24,6 @@
# 6. Add a new "new_http_archive" entry to the WORKSPACE file for the tarball
# you just uploaded.
-# TODO(phil): Deal with armhf packages. Right now only works for amd64.
-
def download_packages(name, packages, excludes = [], force_includes = [], target_compatible_with = None):
"""Downloads a set of packages as well as their dependencies.
@@ -39,19 +39,38 @@
excludes_list = " ".join(["--exclude=%s" % e for e in excludes])
force_includes = " ".join(["--force-include=%s" % i for i in force_includes])
native.genrule(
+ name = name + "_gen",
+ outs = ["%s.sh" % name],
+ executable = True,
+ cmd = """
+cat > $@ <<'END'
+#!/bin/bash
+
+# --- begin runfiles.bash initialization v2 ---
+# Copy-pasted from the Bazel Bash runfiles library v2.
+set -uo pipefail; f=bazel_tools/tools/bash/runfiles/runfiles.bash
+source "$${RUNFILES_DIR:-/dev/null}/$$f" 2>/dev/null || \\
+ source "$$(grep -sm1 "^$$f " "$${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \\
+ source "$$0.runfiles/$$f" 2>/dev/null || \\
+ source "$$(grep -sm1 "^$$f " "$$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \\
+ source "$$(grep -sm1 "^$$f " "$$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \\
+ { echo>&2 "ERROR: cannot find $$f"; exit 1; }; f=; set -e
+# --- end runfiles.bash initialization v2 ---
+
+
+exec "$$(rlocation org_frc971/debian/download_packages)" %s %s %s "$$@"
+END""" % (force_includes, excludes_list, package_list),
+ target_compatible_with = target_compatible_with,
+ )
+ native.sh_binary(
name = name,
- outs = ["%s_output.txt" % name],
- tags = [
- "local",
- "manual",
+ srcs = ["%s.sh" % name],
+ deps = [
+ "@bazel_tools//tools/bash/runfiles",
],
- tools = [
+ data = [
"//debian:download_packages",
],
- # TODO(phil): Deal with stderr a bit better. It spews more stuff out than I
- # would like it to.
- cmd = "$(location //debian:download_packages) %s %s %s | tee $@ >&2" %
- (force_includes, excludes_list, package_list),
target_compatible_with = target_compatible_with,
)