Add infrastructure to download deb packages for the build
See //debian/packages.bzl for a bit more detail.
In this patch I've added the Python packages that we'll use later on
in the sandbox setup.
Change-Id: Ia651d778cc85ecdbf78d50230fea652f7633f799
diff --git a/WORKSPACE b/WORKSPACE
index e1cb1f8..1dc4a36 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -1,5 +1,10 @@
workspace(name = 'org_frc971')
+load("//debian:python.bzl", python_debs="files")
+load("//debian:packages.bzl", "generate_repositories_for_debs")
+
+generate_repositories_for_debs(python_debs)
+
new_local_repository(
name = 'usr_repo',
path = '/usr',
diff --git a/debian/BUILD b/debian/BUILD
index a20b894..845a424 100644
--- a/debian/BUILD
+++ b/debian/BUILD
@@ -1,3 +1,36 @@
-package(default_visibility = ['//visibility:public'])
+package(default_visibility = ["//visibility:public"])
-# TODO(phil): Add deb-to-tarball targets here.
+load(
+ "//debian:python.bzl",
+ python_debs = "files",
+)
+load("//debian:packages.bzl", "download_packages", "generate_deb_tarball")
+
+py_binary(
+ name = "download_packages",
+ srcs = [
+ "download_packages.py",
+ ],
+ default_python_version = "PY3",
+ main = "download_packages.py",
+ srcs_version = "PY2AND3",
+)
+
+download_packages(
+ name = "download_python_deps",
+ excludes = [
+ "libblas.so.3",
+ "liblapack.so.3",
+ ],
+ packages = [
+ "python-dev",
+ "python-numpy",
+ "python3-dev",
+ "python3-numpy",
+ ],
+)
+
+generate_deb_tarball(
+ name = "python",
+ files = python_debs,
+)
diff --git a/debian/download_packages.py b/debian/download_packages.py
new file mode 100755
index 0000000..4709463
--- /dev/null
+++ b/debian/download_packages.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import re
+import subprocess
+import tempfile
+import argparse
+import hashlib
+
+def get_deps(package):
+ out = subprocess.check_output(["apt-rdepends", package])
+ deps = out.splitlines()
+ return set([dep for dep in deps if not dep.startswith(b" ")])
+
+def get_all_deps(packages):
+ deps = set()
+ for package in packages:
+ deps.update(get_deps(package))
+ return deps
+
+def download_deps(packages, excludes):
+ deps = get_all_deps(packages)
+ exclude_deps = get_all_deps(excludes)
+ deps -= exclude_deps
+ subprocess.check_call([b"apt-get", b"download"] + list(deps))
+
+def fixup_files():
+ # Gotta remove those pesky epoch numbers in the file names. Bazel doesn't
+ # like them.
+ regex = re.compile(".%3a")
+ contents = os.listdir(os.getcwd())
+ for deb in contents:
+ new_name = regex.sub("", deb)
+ if new_name != deb:
+ os.rename(deb, new_name)
+
+def sha256_checksum(filename, block_size=65536):
+ sha256 = hashlib.sha256()
+ with open(filename, 'rb') as f:
+ for block in iter(lambda: f.read(block_size), b''):
+ sha256.update(block)
+ return sha256.hexdigest()
+
+def print_file_list():
+ contents = os.listdir(os.getcwd())
+ contents.sort()
+ print("_files = {")
+ for deb in contents:
+ print(' "%s": "%s",' % (deb, sha256_checksum(deb)))
+ print("}")
+
+def main(argv):
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--exclude", "-e", type=str, action="append", help="A package to exclude from the list")
+ parser.add_argument("package", nargs="+", help="The packages to download.")
+ args = parser.parse_args(argv[1:])
+ folder = tempfile.mkdtemp()
+ os.chdir(folder)
+ excludes = args.exclude or []
+ # Exclude common packages that don't make sense to include in everything all
+ # the time.
+ excludes += ["libc-dev", "debconf", "install-info", "debconf-2.0", "libc6", "libc6-dev", "dpkg"]
+ download_deps(args.package, excludes)
+ fixup_files()
+ print_file_list()
+ print("Your packages are all in %s" % folder)
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv))
diff --git a/debian/packages.bzl b/debian/packages.bzl
new file mode 100644
index 0000000..d5603ec
--- /dev/null
+++ b/debian/packages.bzl
@@ -0,0 +1,99 @@
+load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
+
+# In order to use deb packages in the build you have to follow these steps:
+#
+# 1. Create a "download_packages" build step in //debian/BUILD. List the
+# packages you care about and exclude the ones you don't care about.
+# Invoke "bazel build" on the "download_packages" target you just created.
+# Save the "_files" dictionary it prints into a .bzl file in the //debian
+# folder.
+# 2. The "download_packages" steps prints the location of the deb packages
+# after it prints the "_files" dictionary. Take the deb packages from there
+# and upload them to http://frc971.org/Build-Dependencies/.
+# 3. Add the newly uploaded deb packages as WORKSPACE entries using the
+# "generate_repositories_for_debs" helper. Load the "_files" dictionary
+# created earlier and the "generate_repositories_for_debs" helper and call
+# them together in the WORKSPACE file.
+# 4. Add a "generate_deb_tarball" target to //debian/BUILD. Pass in the
+# "_files" dictionary created earlier by loading it from the .bzl file.
+# 5. Invoke "bazel build" on the "generate_deb_tarball" target you just created
+# and upload the resulting tarball to http://frc971.org/Build-Dependencies.
+# 6. Add a new "new_http_archive" entry to the WORKSPACE file for the tarball
+# you just uploaded.
+
+# TODO(phil): Deal with armhf packages. Right now only works for amd64.
+
+def download_packages(name, packages, excludes=[]):
+ """Downloads a set of packages as well as their dependencies.
+
+ You can also specify excludes in case some of the dependencies are meta
+ packages.
+
+ Use "bazel run" on these targets to download the packages and generate the
+ list to use in a .bzl file. Once you have the packages on
+ http://frc971.org/Build-Dependencies/ you can add them to a to
+ combine_packages rule.
+ """
+ package_list = " ".join(packages)
+ excludes_list = " ".join(["--exclude=%s" % e for e in excludes])
+ native.genrule(
+ name = name,
+ outs = ["%s_output.txt" % name],
+ tags = [
+ "local",
+ "manual",
+ ],
+ tools = [
+ "//debian:download_packages",
+ ],
+ # TODO(phil): Deal with stderr a bit better. It spews more stuff out than I
+ # would like it to.
+ cmd = "$(location //debian:download_packages) %s %s | tee $@ >&2" % (excludes_list, package_list),
+ )
+
+def _convert_deb_to_target(deb):
+ """Converts a debian package filename to a valid bazel target name."""
+ target = deb.split('_')[0]
+ target = target.replace('-', '_')
+ target = target.replace('.', '_')
+ target = target.replace(':', '_')
+ target = target.replace('+', 'x')
+ return "deb_%s_repo" % target
+
+def generate_repositories_for_debs(files):
+ """A WORKSPACE helper to add all the deb packages in the dictionary as a repo.
+
+ The files dictionary must be one generated with the "download_packages"
+ helper above.
+ """
+ for f in files.keys():
+ name = _convert_deb_to_target(f)
+ if name not in native.existing_rules():
+ native.http_file(
+ name = name,
+ url = 'http://frc971.org/Build-Dependencies/%s' % f,
+ sha256 = files[f],
+ )
+
+def generate_deb_tarball(name, files):
+ """Takes all debs in the dictionary and generates one tarball from them.
+
+ This can then be uploaded and used as another WORKSPACE entry.
+ """
+ deps = []
+ for f in files.keys():
+ dep = _convert_deb_to_target(f)
+ deps.append(dep)
+ if ('generate_%s_tarball' % dep) not in native.existing_rules():
+ native.genrule(
+ name = 'generate_%s_tarball' % dep,
+ srcs = ['@%s//file' % dep],
+ outs = ['extracted_%s.tar' % dep],
+ cmd = 'dpkg-deb --fsys-tarfile $(SRCS) > $@',
+ )
+
+ pkg_tar(
+ name = name,
+ extension = 'tar.gz',
+ deps = ['extracted_%s.tar' % dep for dep in deps],
+ )
diff --git a/debian/python.bzl b/debian/python.bzl
new file mode 100644
index 0000000..78ed285
--- /dev/null
+++ b/debian/python.bzl
@@ -0,0 +1,46 @@
+files = {
+ "dh-python_1.20141111-2_all.deb": "33672e53f4c255288e3f73117c06347b010b616f221d8d265593299d1b522f2c",
+ "libblas-common_1.2.20110419-10_amd64.deb": "5aea4f73762e70f50bd292e6ee3e3e5d8f16613dc1b2b116ad7c57891ba471cd",
+ "libblas3_1.2.20110419-10_amd64.deb": "06cff144803720414f586491d2533f31e4e6f820cfaabccc8b0fbac81b98e086",
+ "libdb5.3_5.3.28-9+deb8u1_amd64.deb": "9740f05d3c6824911be1d80b71efbccf67a06fd3287d65b1e68666a7d356d51f",
+ "libexpat1-dev_2.1.0-6+deb8u4_amd64.deb": "6b4231b09a96933ff25aae9b68eaaa3b9252b82ca2fd37eccbe2a7823ea6d9ed",
+ "libexpat1_2.1.0-6+deb8u4_amd64.deb": "de7979297d0298271d71b4554772ba4da60ba6895ed86ca8fc9c1159c58913e4",
+ "libffi6_3.1-2+deb8u1_amd64.deb": "100343fca79ff265abc62467c7085fca68b8764e8c2551302ab741c771e7f0aa",
+ "libgfortran3_4.9.2-10_amd64.deb": "77798b64f1f042daca070e3edead2658ffed6a9dcf888ba8e22f6f140012510c",
+ "liblapack3_3.5.0-4_amd64.deb": "d5b06e70f99ca0389b29b334a6d30d7ecddd08d2d11d0cb0b9bdcf7e230ce1f7",
+ "libmpdec2_2.4.1-1_amd64.deb": "b61ae05899abfb6b82af1915e33ae72e0b7caf8035416fbbcb8128832fcb26d2",
+ "libncursesw5_5.9+20140913-1+deb8u2_amd64.deb": "9d8d80d077e0ca85ac4493173431df42fe8943a457bac35625433eb414d79eca",
+ "libpython-dev_2.7.9-1_amd64.deb": "de5b306431959a50a8368f292157ee573aac86fa7e88cacd6b03983cf85570c4",
+ "libpython-stdlib_2.7.9-1_amd64.deb": "5f9ffb96222498c764526a83cac48281a941ec6e8470db1a1f8e17e6236a5669",
+ "libpython2.7-dev_2.7.9-2+deb8u1_amd64.deb": "af8754ad818d600d39cf6ab878f5afeb7265fa51da44487f21ab06e7df6462cd",
+ "libpython2.7-minimal_2.7.9-2+deb8u1_amd64.deb": "916e2c541aa954239cb8da45d1d7e4ecec232b24d3af8982e76bf43d3e1758f3",
+ "libpython2.7-stdlib_2.7.9-2+deb8u1_amd64.deb": "cf1c9dfc12d6cfd42bb14bfb46ee3cec0f6ebc720a1419f017396739953b12c5",
+ "libpython2.7_2.7.9-2+deb8u1_amd64.deb": "cfb120644cfcb4e08a77069c388197c4db4fbf0136c80973d0194cc562248904",
+ "libpython3-dev_3.4.2-2_amd64.deb": "7a7574bcec9163fdd16e76c8a45e0eac5c139efc1df81dcefab99c65fcf507a3",
+ "libpython3-stdlib_3.4.2-2_amd64.deb": "a56d9ec1e5f9ab0c51d169f0b332164b3d6cd10b7ae1025688305239e8781dcc",
+ "libpython3.4-dev_3.4.2-1_amd64.deb": "760e995b0676eac09194554c235bcc164f6cff54fd2c09603f16a67ad6d9874b",
+ "libpython3.4-minimal_3.4.2-1_amd64.deb": "d4c9fa2e127ca7799930152aedc37c3d891e0ee439e0595f0a654678570bfa34",
+ "libpython3.4-stdlib_3.4.2-1_amd64.deb": "afa4d641e50b5671230f92f2a8bcf8ee961fbb54c9c3a9656a8ec9fd0391765b",
+ "libpython3.4_3.4.2-1_amd64.deb": "089d804a6432d7a321cfd5a95d191251fa3a0d324451981e29ff695f765b5c06",
+ "libquadmath0_4.9.2-10_amd64.deb": "76b71fdb834434e7b6dde5ba343af9bacddb987ef8ad9c788442dbe4e236e78f",
+ "libreadline6_6.3-8+b3_amd64.deb": "647948737fcfea4749368aa233b2d8b89032546ba4db2f0338239e9a7f4bda3e",
+ "libsqlite3-0_3.8.7.1-1+deb8u2_amd64.deb": "969b13188c642196def3846e1e44e7923bcf1fa07374b0fd7fe766ea2ba11bd0",
+ "libssl1.0.0_1.0.1t-1+deb8u7_amd64.deb": "d99de2cdca54484d23badc5683c7211b3a191977272d9e5281837af863dcdd56",
+ "libtinfo5_5.9+20140913-1+deb8u2_amd64.deb": "914cb3f1f52425ecd92c44aacdb3b1303b57db783ad53910c2bb1725a56ffbaf",
+ "mime-support_3.58_all.deb": "c05ebe8f38da4ff19d028c9f4680414149e5c7a746de13bc9db0a562796ed213",
+ "python-dev_2.7.9-1_amd64.deb": "402f2b3185fb83be92e9d13a08ed1f7678adf72c8bade7ccdec2d47d3321c5ae",
+ "python-minimal_2.7.9-1_amd64.deb": "58649e19c19847e264b32ad8cb5a8f58477829e1afc2616c54cb0a1ef30496be",
+ "python-numpy_1.8.2-2_amd64.deb": "598926db22f9f16a3f0597fdfcbdc3c271ef4991223ca3166bd5dbcb775a7831",
+ "python2.7-dev_2.7.9-2+deb8u1_amd64.deb": "1d423c1f15f9ac0cf3d3c69958cc93cc7b673d39ac9200c1de84b82eb4be4a8d",
+ "python2.7-minimal_2.7.9-2+deb8u1_amd64.deb": "c89199f908d5a508d8d404efc0e1aef3d9db59ea23bd4532df9e59941643fcfb",
+ "python2.7_2.7.9-2+deb8u1_amd64.deb": "00c99c8dc1cda85053c8bfc7ea34ae5c40408c54b498ca22d0e2cb6b0acb796c",
+ "python3-dev_3.4.2-2_amd64.deb": "c94a0b57c74e6158cde842e6376ee614fbefb380ab1e1bbed66b176b87090ed5",
+ "python3-minimal_3.4.2-2_amd64.deb": "0a8f9f1e824929d6c7412538e1a7fa4f56c8d68565cf3aba3cbefe05a4187c8b",
+ "python3-numpy_1.8.2-2_amd64.deb": "7e514578bee0eabee43915185d73526b0e28b912a31aa665920fbec16db380fc",
+ "python3.4-dev_3.4.2-1_amd64.deb": "bc20dad65f0c37c712d612e247a8510888cbc97568659a00b45b2a0915e4e4b4",
+ "python3.4-minimal_3.4.2-1_amd64.deb": "a2c868cd2deaa8467aa6fb4bfc2ff17001418de163195a86d02ae16c656ec373",
+ "python3.4_3.4.2-1_amd64.deb": "398a1bf2c0c7c8f7271b9150b8db61f225c424b96fe2befcac9abea76a793d74",
+ "python3_3.4.2-2_amd64.deb": "ce6e42f5d87103ddb799f6b025ca3fe1e57a509e7303636e127a83eabef2ab2c",
+ "python_2.7.9-1_amd64.deb": "93dc9d03df366d658832fb238a6c1e6c57e5e57dd648145c2f57a8f3e2b419ed",
+ "readline-common_6.3-8_all.deb": "8b91bce988c38798e565820919a600f1a58ca483d8406860cc37e847a55a6bfd",
+}