Import gazelle
This patch imports gazelle as a linter. It automatically generates
BUILD file entries for Go code and at the same time keeps BUILD files
formatted.
The `tools/lint:run-ci` target is set up to automatically add new Go
repositories as well.
I added a tool at `//tools/go:mirror_go_repos` that needs to be run
before anyone can merge code that uses third-party Go libraries.
Change-Id: I1fbf6761439d45893f5be88d294ccc3c567840ca
Signed-off-by: Philipp Schrader <philipp.schrader@gmail.com>
diff --git a/tools/go/mirror_go_repos.py b/tools/go/mirror_go_repos.py
new file mode 100644
index 0000000..dc160fa
--- /dev/null
+++ b/tools/go/mirror_go_repos.py
@@ -0,0 +1,144 @@
+"""This script mirrors the dependencies from go_deps.bzl as Build-Dependencies.
+
+We use "go mod download" to manually download each Go dependency. We then tar
+up all the dependencies and copy them to the Build-Dependencies server for
+hosting.
+"""
+
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tarfile
+from typing import List, Dict
+import urllib.request
+
+import tools.go.mirror_lib
+
+GO_DEPS_WWWW_DIR = "/var/www/html/files/frc971/Build-Dependencies/go_deps"
+
+def compute_sha256(filepath: str) -> str:
+ """Computes the SHA256 of a file at the specified location."""
+ with open(filepath, "rb") as file:
+ contents = file.read()
+ return hashlib.sha256(contents).hexdigest()
+
+def get_existing_mirrored_repos(ssh_host: str) -> Dict[str, str]:
+ """Gathers information about the libraries that are currently mirrored."""
+ run_result = subprocess.run(["ssh", ssh_host, f"bash -c 'sha256sum {GO_DEPS_WWWW_DIR}/*'"], check=True, stdout=subprocess.PIPE)
+
+ existing_mirrored_repos = {}
+ for line in run_result.stdout.decode("utf-8").splitlines():
+ sha256, fullpath = line.split()
+ existing_mirrored_repos[Path(fullpath).name] = sha256
+
+ return existing_mirrored_repos
+
+def download_repos(
+ repos: Dict[str, str],
+ existing_mirrored_repos: Dict[str, str],
+ tar: tarfile.TarFile) -> Dict[str, str]:
+ """Downloads the not-yet-mirrored repos into a tarball."""
+ cached_info = {}
+
+ for repo in repos:
+ print(f"Downloading file for {repo['name']}")
+ importpath = repo["importpath"]
+ version = repo["version"]
+ module = f"{importpath}@{version}"
+
+ download_result = subprocess.run(
+ ["external/go_sdk/bin/go", "mod", "download", "-json", module],
+ check=True, stdout=subprocess.PIPE)
+ if download_result.returncode != 0:
+ print("Failed to download file.")
+ return 1
+
+ module_info = json.loads(download_result.stdout.decode("utf-8"))
+
+ name = repo["name"]
+ zip_path = Path(module_info["Zip"])
+ mirrored_name = f"{name}__{zip_path.name}"
+ if mirrored_name not in existing_mirrored_repos:
+ # We only add the Go library to the tarball if it's not already
+ # mirrored. We don't want to overwrite files.
+ tar.add(zip_path, arcname=mirrored_name)
+ sha256 = compute_sha256(zip_path)
+ else:
+ # Use the already-computed checksum for consistency.
+ sha256 = existing_mirrored_repos[mirrored_name]
+
+ cached_info[name] = {
+ "strip_prefix": module,
+ "filename": mirrored_name,
+ "sha256": sha256,
+ "version": version,
+ "importpath": importpath,
+ }
+
+ return cached_info
+
+def copy_to_host_and_unpack(filename: str, ssh_host: str) -> None:
+ subprocess.run(["scp", filename, f"{ssh_host}:"], check=True)
+
+ # Be careful not to use single quotes in these commands to avoid breaking
+ # the subprocess.run() invocation below.
+ command = " && ".join([
+ f"tar -C {GO_DEPS_WWWW_DIR} --no-same-owner -xvaf {filename}",
+ # Change the permissions so other users can read them (and checksum
+ # them).
+ f"find {GO_DEPS_WWWW_DIR}/ -type f -exec chmod 644 {{}} +",
+ ])
+
+ print("You might be asked for your sudo password shortly.")
+ subprocess.run(["ssh", "-t", ssh_host, f"sudo -u www-data bash -c '{command}'"], check=True)
+
+def main(argv):
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--ssh_host",
+ type=str,
+ help=("The SSH host to copy the downloaded Go repositories to. This "
+ "should be software.971spartans.net where all the "
+ "Build-Dependencies files live. Only specify this if you have "
+ "access to the server."))
+ parser.add_argument("--go_deps_bzl", type=str, default="go_deps.bzl")
+ parser.add_argument("--go_mirrors_bzl", type=str, default="tools/go/go_mirrors.bzl")
+ args = parser.parse_args(argv[1:])
+
+ os.chdir(os.environ["BUILD_WORKSPACE_DIRECTORY"])
+
+ repos = tools.go.mirror_lib.parse_go_repositories(args.go_deps_bzl)
+
+ if args.ssh_host:
+ existing_mirrored_repos = get_existing_mirrored_repos(args.ssh_host)
+ else:
+ existing_mirrored_repos = {}
+
+ with tarfile.open("go_deps.tar", "w") as tar:
+ cached_info = download_repos(repos, existing_mirrored_repos, tar)
+ num_not_already_mirrored = len(tar.getnames())
+
+ print(f"Found {num_not_already_mirrored}/{len(cached_info)} libraries "
+ "that need to be mirrored.")
+
+ # Only mirror the deps if we've specified an SSH host and we actually have
+ # something to mirror.
+ if args.ssh_host and num_not_already_mirrored:
+ copy_to_host_and_unpack("go_deps.tar", args.ssh_host)
+ else:
+ print("Skipping mirroring because of lack of --ssh_host or there's "
+ "nothing to actually mirror.")
+
+ with open(args.go_mirrors_bzl, "w") as file:
+ file.write("# This file is auto-generated. Do not edit.\n")
+ file.write("GO_MIRROR_INFO = ")
+ json.dump(cached_info, file, indent=2, sort_keys=True)
+ file.write("\n")
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv))