Philipp Schrader | 9670a2a | 2023-01-11 01:20:16 -0800 | [diff] [blame] | 1 | load("@rules_pkg//:pkg.bzl", "pkg_tar") |
Austin Schuh | f093fa5 | 2019-06-23 20:47:23 -0700 | [diff] [blame] | 2 | load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") |
Philipp Schrader | 0e19c60 | 2018-03-07 21:07:22 -0800 | [diff] [blame] | 3 | |
Stephan Pleines | 69a52cb | 2023-12-14 19:13:51 -0800 | [diff] [blame] | 4 | # In order to use deb packages in the build you have to follow these steps. |
Philipp Schrader | 0e19c60 | 2018-03-07 21:07:22 -0800 | [diff] [blame] | 5 | # |
Stephan Pleines | 69a52cb | 2023-12-14 19:13:51 -0800 | [diff] [blame] | 6 | # Adding new packages: |
Philipp Schrader | 0e19c60 | 2018-03-07 21:07:22 -0800 | [diff] [blame] | 7 | # 1. Create a "download_packages" build step in //debian/BUILD. List the |
| 8 | # packages you care about and exclude the ones you don't care about. |
Brian Silverman | 4b90dd0 | 2022-01-08 18:16:22 -0800 | [diff] [blame] | 9 | # Invoke "bazel run" on the "download_packages" target you just created. |
Philipp Schrader | 0e19c60 | 2018-03-07 21:07:22 -0800 | [diff] [blame] | 10 | # Save the "_files" dictionary it prints into a .bzl file in the //debian |
James Kuszmaul | 3ae4226 | 2019-11-08 12:33:41 -0800 | [diff] [blame] | 11 | # folder. You will need to have the apt-rdepends package installed. |
Brian Silverman | 4b90dd0 | 2022-01-08 18:16:22 -0800 | [diff] [blame] | 12 | # If you want to get packages for a different architecture or distribution, |
| 13 | # you can pass flags here to control those. See the --help for details. |
Philipp Schrader | 0e19c60 | 2018-03-07 21:07:22 -0800 | [diff] [blame] | 14 | # 2. The "download_packages" steps prints the location of the deb packages |
| 15 | # after it prints the "_files" dictionary. Take the deb packages from there |
Maxwell Henderson | 7a93a65 | 2023-06-24 15:17:09 -0700 | [diff] [blame] | 16 | # and upload them to https://software.frc971.org/Build-Dependencies/. |
Philipp Schrader | 0e19c60 | 2018-03-07 21:07:22 -0800 | [diff] [blame] | 17 | # 3. Add the newly uploaded deb packages as WORKSPACE entries using the |
| 18 | # "generate_repositories_for_debs" helper. Load the "_files" dictionary |
| 19 | # created earlier and the "generate_repositories_for_debs" helper and call |
| 20 | # them together in the WORKSPACE file. |
| 21 | # 4. Add a "generate_deb_tarball" target to //debian/BUILD. Pass in the |
| 22 | # "_files" dictionary created earlier by loading it from the .bzl file. |
| 23 | # 5. Invoke "bazel build" on the "generate_deb_tarball" target you just created |
Maxwell Henderson | 7a93a65 | 2023-06-24 15:17:09 -0700 | [diff] [blame] | 24 | # and upload the resulting tarball to https://software.frc971.org/Build-Dependencies. |
Philipp Schrader | 0e19c60 | 2018-03-07 21:07:22 -0800 | [diff] [blame] | 25 | # 6. Add a new "new_http_archive" entry to the WORKSPACE file for the tarball |
| 26 | # you just uploaded. |
Stephan Pleines | 69a52cb | 2023-12-14 19:13:51 -0800 | [diff] [blame] | 27 | # |
| 28 | # Updating existing packages: |
| 29 | # 1. Read above instructions. |
| 30 | # 2. The "download_packages" build step already exists, run it. |
| 31 | # 3. The .bzl file with the file list already exists. Update it with the |
| 32 | # output from the previous step. |
| 33 | # 4. Follow steps 2., 5., and 6. from "adding new packages". |
Philipp Schrader | 0e19c60 | 2018-03-07 21:07:22 -0800 | [diff] [blame] | 34 | |
Alexei Strots | f3b7e8d | 2023-08-16 21:35:13 -0700 | [diff] [blame] | 35 | def download_packages(name, packages, excludes = [], force_includes = [], force_excludes = [], target_compatible_with = None, release = "bullseye"): |
Austin Schuh | 8e17be9 | 2019-12-24 09:32:11 -0800 | [diff] [blame] | 36 | """Downloads a set of packages as well as their dependencies. |
Philipp Schrader | 0e19c60 | 2018-03-07 21:07:22 -0800 | [diff] [blame] | 37 | |
Austin Schuh | 8e17be9 | 2019-12-24 09:32:11 -0800 | [diff] [blame] | 38 | You can also specify excludes in case some of the dependencies are meta |
| 39 | packages. |
Philipp Schrader | 0e19c60 | 2018-03-07 21:07:22 -0800 | [diff] [blame] | 40 | |
Austin Schuh | 8e17be9 | 2019-12-24 09:32:11 -0800 | [diff] [blame] | 41 | Use "bazel run" on these targets to download the packages and generate the |
| 42 | list to use in a .bzl file. Once you have the packages on |
Maxwell Henderson | 7a93a65 | 2023-06-24 15:17:09 -0700 | [diff] [blame] | 43 | https://software.frc971.org/Build-Dependencies/ you can add them to a to |
Austin Schuh | 8e17be9 | 2019-12-24 09:32:11 -0800 | [diff] [blame] | 44 | combine_packages rule. |
Philipp Schrader | fd5489f | 2022-09-17 17:31:09 -0700 | [diff] [blame] | 45 | |
| 46 | force_includes lets you include packages that are excluded by default. The |
| 47 | dependencies of these force-included packages are also force-included. To |
| 48 | counter-act that, you can use "force_excludes". The force-excluded packages |
| 49 | are excluded even if they're pulled in as a dependency from a |
| 50 | "force_includes" package. |
Austin Schuh | 8e17be9 | 2019-12-24 09:32:11 -0800 | [diff] [blame] | 51 | """ |
| 52 | package_list = " ".join(packages) |
| 53 | excludes_list = " ".join(["--exclude=%s" % e for e in excludes]) |
| 54 | force_includes = " ".join(["--force-include=%s" % i for i in force_includes]) |
Philipp Schrader | fd5489f | 2022-09-17 17:31:09 -0700 | [diff] [blame] | 55 | force_excludes = " ".join(["--force-exclude=%s" % e for e in force_excludes]) |
Austin Schuh | 8e17be9 | 2019-12-24 09:32:11 -0800 | [diff] [blame] | 56 | native.genrule( |
Brian Silverman | 4b90dd0 | 2022-01-08 18:16:22 -0800 | [diff] [blame] | 57 | name = name + "_gen", |
| 58 | outs = ["%s.sh" % name], |
| 59 | executable = True, |
| 60 | cmd = """ |
| 61 | cat > $@ <<'END' |
| 62 | #!/bin/bash |
| 63 | |
| 64 | # --- begin runfiles.bash initialization v2 --- |
| 65 | # Copy-pasted from the Bazel Bash runfiles library v2. |
| 66 | set -uo pipefail; f=bazel_tools/tools/bash/runfiles/runfiles.bash |
| 67 | source "$${RUNFILES_DIR:-/dev/null}/$$f" 2>/dev/null || \\ |
| 68 | source "$$(grep -sm1 "^$$f " "$${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \\ |
| 69 | source "$$0.runfiles/$$f" 2>/dev/null || \\ |
| 70 | source "$$(grep -sm1 "^$$f " "$$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \\ |
| 71 | source "$$(grep -sm1 "^$$f " "$$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \\ |
| 72 | { echo>&2 "ERROR: cannot find $$f"; exit 1; }; f=; set -e |
| 73 | # --- end runfiles.bash initialization v2 --- |
| 74 | |
| 75 | |
Alexei Strots | f3b7e8d | 2023-08-16 21:35:13 -0700 | [diff] [blame] | 76 | exec "$$(rlocation org_frc971/debian/download_packages)" %s %s %s %s --release=%s "$$@" |
| 77 | END""" % (force_includes, force_excludes, excludes_list, package_list, release), |
Brian Silverman | 4b90dd0 | 2022-01-08 18:16:22 -0800 | [diff] [blame] | 78 | target_compatible_with = target_compatible_with, |
| 79 | ) |
| 80 | native.sh_binary( |
Austin Schuh | 8e17be9 | 2019-12-24 09:32:11 -0800 | [diff] [blame] | 81 | name = name, |
Brian Silverman | 4b90dd0 | 2022-01-08 18:16:22 -0800 | [diff] [blame] | 82 | srcs = ["%s.sh" % name], |
| 83 | deps = [ |
| 84 | "@bazel_tools//tools/bash/runfiles", |
Austin Schuh | 8e17be9 | 2019-12-24 09:32:11 -0800 | [diff] [blame] | 85 | ], |
Brian Silverman | 4b90dd0 | 2022-01-08 18:16:22 -0800 | [diff] [blame] | 86 | data = [ |
Austin Schuh | 8e17be9 | 2019-12-24 09:32:11 -0800 | [diff] [blame] | 87 | "//debian:download_packages", |
| 88 | ], |
Philipp Schrader | dada107 | 2020-11-24 11:34:46 -0800 | [diff] [blame] | 89 | target_compatible_with = target_compatible_with, |
Austin Schuh | 8e17be9 | 2019-12-24 09:32:11 -0800 | [diff] [blame] | 90 | ) |
Philipp Schrader | 0e19c60 | 2018-03-07 21:07:22 -0800 | [diff] [blame] | 91 | |
| 92 | def _convert_deb_to_target(deb): |
Brian Silverman | dc7d805 | 2020-01-31 17:44:30 -0800 | [diff] [blame] | 93 | """Converts a debian package filename to a valid bazel target name.""" |
| 94 | target = deb |
| 95 | target = target.replace("-", "_") |
| 96 | target = target.replace(".", "_") |
| 97 | target = target.replace(":", "_") |
| 98 | target = target.replace("+", "x") |
| 99 | target = target.replace("~", "_") |
| 100 | return "deb_%s_repo" % target |
Philipp Schrader | 0e19c60 | 2018-03-07 21:07:22 -0800 | [diff] [blame] | 101 | |
Maxwell Henderson | 7a93a65 | 2023-06-24 15:17:09 -0700 | [diff] [blame] | 102 | def generate_repositories_for_debs(files, base_url = "https://software.frc971.org/Build-Dependencies"): |
Austin Schuh | 8e17be9 | 2019-12-24 09:32:11 -0800 | [diff] [blame] | 103 | """A WORKSPACE helper to add all the deb packages in the dictionary as a repo. |
Philipp Schrader | 0e19c60 | 2018-03-07 21:07:22 -0800 | [diff] [blame] | 104 | |
Austin Schuh | 8e17be9 | 2019-12-24 09:32:11 -0800 | [diff] [blame] | 105 | The files dictionary must be one generated with the "download_packages" |
| 106 | helper above. |
| 107 | """ |
| 108 | for f in files.keys(): |
| 109 | name = _convert_deb_to_target(f) |
| 110 | if name not in native.existing_rules(): |
| 111 | http_file( |
| 112 | name = name, |
| 113 | urls = [base_url + "/" + f], |
| 114 | sha256 = files[f], |
| 115 | downloaded_file_path = f, |
| 116 | ) |
Philipp Schrader | 0e19c60 | 2018-03-07 21:07:22 -0800 | [diff] [blame] | 117 | |
Philipp Schrader | dada107 | 2020-11-24 11:34:46 -0800 | [diff] [blame] | 118 | def generate_deb_tarball(name, files, target_compatible_with = None): |
Austin Schuh | 8e17be9 | 2019-12-24 09:32:11 -0800 | [diff] [blame] | 119 | """Takes all debs in the dictionary and generates one tarball from them. |
Philipp Schrader | 0e19c60 | 2018-03-07 21:07:22 -0800 | [diff] [blame] | 120 | |
Austin Schuh | 8e17be9 | 2019-12-24 09:32:11 -0800 | [diff] [blame] | 121 | This can then be uploaded and used as another WORKSPACE entry. |
| 122 | """ |
| 123 | deps = [] |
| 124 | for f in files.keys(): |
| 125 | dep = _convert_deb_to_target(f) |
| 126 | deps.append(dep) |
| 127 | if ("generate_%s_tarball" % dep) not in native.existing_rules(): |
| 128 | native.genrule( |
| 129 | name = "generate_%s_tarball" % dep, |
| 130 | srcs = ["@%s//file" % dep], |
| 131 | outs = ["extracted_%s.tar" % dep], |
| 132 | cmd = "dpkg-deb --fsys-tarfile $(SRCS) > $@", |
Philipp Schrader | dada107 | 2020-11-24 11:34:46 -0800 | [diff] [blame] | 133 | target_compatible_with = target_compatible_with, |
Austin Schuh | 8e17be9 | 2019-12-24 09:32:11 -0800 | [diff] [blame] | 134 | ) |
Philipp Schrader | 0e19c60 | 2018-03-07 21:07:22 -0800 | [diff] [blame] | 135 | |
Austin Schuh | 8e17be9 | 2019-12-24 09:32:11 -0800 | [diff] [blame] | 136 | pkg_tar( |
| 137 | name = name, |
| 138 | extension = "tar.gz", |
| 139 | deps = ["extracted_%s.tar" % dep for dep in deps], |
Philipp Schrader | dada107 | 2020-11-24 11:34:46 -0800 | [diff] [blame] | 140 | target_compatible_with = target_compatible_with, |
Austin Schuh | 8e17be9 | 2019-12-24 09:32:11 -0800 | [diff] [blame] | 141 | ) |