| #!/usr/bin/env python3 |
| |
| from collections import namedtuple |
| import sys |
| import os |
| import os.path |
| import re |
| import subprocess |
| import tempfile |
| import urllib.request |
| import argparse |
| import hashlib |
| |
| DistroInfo = namedtuple('DistroInfo', |
| ['sources_list', 'url_keys', 'public_keys']) |
| |
| _DISTROS = { |
| "ubuntu": |
| DistroInfo( |
| """deb http://us.archive.ubuntu.com/ubuntu/ {release} main restricted |
| deb-src http://us.archive.ubuntu.com/ubuntu/ {release} main restricted |
| |
| deb http://security.ubuntu.com/ubuntu {release}-security main restricted |
| deb-src http://security.ubuntu.com/ubuntu {release}-security main restricted |
| |
| deb http://us.archive.ubuntu.com/ubuntu/ {release} universe |
| deb-src http://us.archive.ubuntu.com/ubuntu/ {release} universe |
| deb http://us.archive.ubuntu.com/ubuntu/ {release}-updates universe |
| deb-src http://us.archive.ubuntu.com/ubuntu/ {release}-updates universe |
| |
| deb http://us.archive.ubuntu.com/ubuntu/ {release}-updates main restricted |
| deb-src http://us.archive.ubuntu.com/ubuntu/ {release}-updates main restricted |
| |
| deb http://us.archive.ubuntu.com/ubuntu {release}-backports main restricted |
| deb-src http://us.archive.ubuntu.com/ubuntu {release}-backports main restricted""", |
| [], ['3B4FE6ACC0B21F32', '871920D1991BC93C']), |
| "debian": |
| DistroInfo( |
| """deb http://deb.debian.org/debian/ {release} main contrib non-free |
| deb-src http://deb.debian.org/debian/ {release} main contrib non-free |
| |
| deb https://security.debian.org/debian-security {release}-security main contrib non-free |
| deb-src https://security.debian.org/debian-security {release}-security main contrib non-free |
| |
| deb http://deb.debian.org/debian/ {release}-updates main contrib non-free |
| deb-src http://deb.debian.org/debian/ {release}-updates main contrib non-free |
| |
| deb http://deb.debian.org/debian {release}-backports main contrib non-free |
| deb-src http://deb.debian.org/debian {release}-backports main contrib non-free""", |
| [ |
| "https://ftp-master.debian.org/keys/archive-key-11.asc", |
| "https://ftp-master.debian.org/keys/archive-key-11-security.asc" |
| ], []), |
| } |
| |
| |
| def initialize_apt(apt_dir, apt_args, args): |
| os.mkdir(os.path.join(apt_dir, 'etc')) |
| os.mkdir(os.path.join(apt_dir, 'etc', 'apt')) |
| os.mkdir(os.path.join(apt_dir, 'etc', 'apt', 'trusted.gpg.d')) |
| os.mkdir(os.path.join(apt_dir, 'etc', 'apt', 'preferences.d')) |
| os.mkdir(os.path.join(apt_dir, 'var')) |
| os.mkdir(os.path.join(apt_dir, 'var', 'lib')) |
| os.mkdir(os.path.join(apt_dir, 'var', 'lib', 'dpkg')) |
| with open(os.path.join(apt_dir, 'var', 'lib', 'dpkg', 'status'), 'w'): |
| pass |
| distro = _DISTROS[args.distro] |
| with open(os.path.join(apt_dir, 'etc', 'apt', 'sources.list'), 'w') as f: |
| f.write(distro.sources_list.format(release=args.release)) |
| for key in args.apt_key + distro.url_keys: |
| basename = os.path.basename(key) |
| urllib.request.urlretrieve( |
| key, os.path.join(apt_dir, 'etc', 'apt', 'trusted.gpg.d', |
| basename)) |
| for key in distro.public_keys: |
| subprocess.check_call([ |
| "apt-key", "--keyring", |
| os.path.join(apt_dir, 'etc', 'apt', 'trusted.gpg'), "adv", |
| "--keyserver", "keyserver.ubuntu.com", "--recv-key", key |
| ]) |
| |
| subprocess.check_call(["apt-get"] + apt_args + ["update"]) |
| |
| |
| def get_deps(apt_args, package): |
| env = dict(os.environ) |
| if 'LD_LIBRARY_PATH' in env: |
| del env['LD_LIBRARY_PATH'] |
| out = subprocess.check_output(["apt-rdepends"] + apt_args + [package], |
| env=env) |
| deps = out.splitlines() |
| return set([dep for dep in deps if not dep.startswith(b" ")]) |
| |
| |
| def get_all_deps(apt_args, packages): |
| deps = set() |
| for package in packages or (): |
| deps.update(get_deps(apt_args, package)) |
| return deps |
| |
| |
| def map_virtual_packages(packages): |
| '''Maps known virtual packages to the preferred concrete packages which |
| provide them.''' |
| for package in packages: |
| if package == b'python-numpy-abi9': |
| yield b'python-numpy' |
| continue |
| if package == b'python3-numpy-abi9': |
| yield b'python3-numpy' |
| continue |
| if package == b'libjack-0.125': |
| yield b'libjack-jackd2-0' |
| continue |
| if package == b'fonts-freefont': |
| yield b'fonts-freefont-ttf' |
| continue |
| if package == b'gsettings-backend': |
| yield b'dconf-gsettings-backend' |
| continue |
| if package == b'gdal-abi-2-4-0': |
| yield b'libgdal20' |
| continue |
| if package == b'libglu1': |
| yield b'libglu1-mesa' |
| continue |
| if package == b'liblapack.so.3': |
| yield b'liblapack3' |
| continue |
| if package == b'libopencl1': |
| yield b'ocl-icd-libopencl1' |
| continue |
| if package == b'libgcc1': |
| yield b'libgcc-s1' |
| continue |
| if package == b'libopencl-1.2-1': |
| yield b'ocl-icd-libopencl1' |
| continue |
| if package == b'libblas.so.3': |
| yield b'libblas3' |
| continue |
| if package == b'debconf-2.0': |
| yield b'debconf' |
| continue |
| yield package |
| |
| |
| def download_deps(apt_args, packages, excludes, force_includes, |
| force_excludes): |
| deps = get_all_deps(apt_args, packages) |
| exclude_deps = get_all_deps(apt_args, excludes) |
| exclude_deps -= set({b'libssl1.1'}) |
| deps -= exclude_deps |
| force_include_deps = get_all_deps(apt_args, force_includes) |
| deps |= force_include_deps |
| force_exclude_deps = get_all_deps(apt_args, force_excludes) |
| deps -= force_exclude_deps |
| env = dict(os.environ) |
| if 'LD_LIBRARY_PATH' in env: |
| del env['LD_LIBRARY_PATH'] |
| subprocess.check_call([b"apt-get"] + [a.encode('utf-8') |
| for a in apt_args] + [b"download"] + |
| list(map_virtual_packages(deps)), |
| env=env) |
| |
| |
| def fixup_files(): |
| # Gotta remove those pesky epoch numbers in the file names. Bazel doesn't |
| # like them. |
| regex = re.compile(".%3a") |
| contents = os.listdir(os.getcwd()) |
| for deb in contents: |
| new_name = regex.sub("", deb) |
| if new_name != deb: |
| os.rename(deb, new_name) |
| |
| |
| def sha256_checksum(filename, block_size=65536): |
| sha256 = hashlib.sha256() |
| with open(filename, 'rb') as f: |
| for block in iter(lambda: f.read(block_size), b''): |
| sha256.update(block) |
| return sha256.hexdigest() |
| |
| |
| def print_file_list(): |
| contents = os.listdir(os.getcwd()) |
| contents.sort() |
| print("_files = {") |
| for deb in contents: |
| print(' "%s": "%s",' % (deb, sha256_checksum(deb))) |
| print("}") |
| |
| |
| _ALWAYS_EXCLUDE = [ |
| "dbus-session-bus", |
| "debconf", |
| "debconf-2.0", |
| "default-dbus-session-bus", |
| "dpkg", |
| "install-info", |
| "libc-dev", |
| "libc6", |
| "libc6-dev", |
| ] |
| |
| |
| def main(argv): |
| parser = argparse.ArgumentParser() |
| parser.add_argument("--exclude", |
| "-e", |
| type=str, |
| action="append", |
| help="A package to exclude from the list") |
| parser.add_argument( |
| "--force-include", |
| type=str, |
| action="append", |
| help= |
| "Force include this and its dependencies. Even if listed in excludes.") |
| parser.add_argument( |
| "--force-exclude", |
| type=str, |
| action="append", |
| help= |
| "Force exclude this and its dependencies. Even if listed via --force-include." |
| ) |
| parser.add_argument("--arch", |
| type=str, |
| default="amd64", |
| help="Architecture to download files for.") |
| parser.add_argument("--distro", |
| type=str, |
| default="debian", |
| choices=_DISTROS.keys(), |
| help="Architecture to download files for.") |
| parser.add_argument( |
| "--apt-dir", |
| type=str, |
| help=" ".join([ |
| "File to generate and store apt files in.", |
| "Helpful for saving time when downloading multiple groups of packages.", |
| "Some flags will be ignored in favor of the values used to create this folder, so be careful.", |
| ])) |
| parser.add_argument("--release", |
| type=str, |
| default="bookworm", |
| help="Debian/Ubuntu release to use.") |
| parser.add_argument("--apt-key", |
| type=str, |
| action="append", |
| default=[], |
| help="URL of an additional apt archive key to trust.") |
| parser.add_argument("package", nargs="+", help="The packages to download.") |
| args = parser.parse_args(argv[1:]) |
| if args.apt_dir: |
| apt_dir = args.apt_dir |
| else: |
| apt_dir = tempfile.mkdtemp() |
| apt_args = ["-o", "Dir=" + apt_dir, "-o", "APT::Architecture=" + args.arch] |
| if not args.apt_dir: |
| print("Creating apt files in %s" % apt_dir) |
| initialize_apt(apt_dir, apt_args, args) |
| folder = tempfile.mkdtemp() |
| os.chdir(folder) |
| excludes = args.exclude or [] |
| # Exclude common packages that don't make sense to include in everything all |
| # the time. |
| excludes += _ALWAYS_EXCLUDE |
| download_deps(apt_args, args.package, excludes, args.force_include, |
| args.force_exclude) |
| fixup_files() |
| print_file_list() |
| print("Your packages are all in %s" % folder) |
| |
| |
| if __name__ == "__main__": |
| sys.exit(main(sys.argv)) |