Automatically prune unused Go dependencies
The way gazelle was set up meant that it would never prune any
dependencies from go_deps.bzl. I'm not exactly sure _why_, but that's
the way it behaved.
The new approach here in this patch is to essentially null out
go_deps.bzl and have gazelle regenerate it from scratch.
Signed-off-by: Philipp Schrader <philipp.schrader@gmail.com>
Change-Id: Ic94020df816afa0df798b19b16b11272579793be
diff --git a/tools/go/BUILD b/tools/go/BUILD
index 25b3adb..2c251a0 100644
--- a/tools/go/BUILD
+++ b/tools/go/BUILD
@@ -6,6 +6,13 @@
srcs = [
"mirror_lib.py",
],
+ data = [
+ "@com_github_bazelbuild_buildtools//buildifier",
+ ],
+ target_compatible_with = ["@platforms//cpu:x86_64"],
+ deps = [
+ "@bazel_tools//tools/python/runfiles",
+ ],
)
py_binary(
@@ -25,13 +32,12 @@
"mirror_go_repos.py",
],
data = [
- "@com_github_bazelbuild_buildtools//buildifier",
"@go_sdk//:bin/go",
],
target_compatible_with = ["@platforms//cpu:x86_64"],
+ visibility = ["//visibility:public"],
deps = [
":mirror_lib",
- "@bazel_tools//tools/python/runfiles",
],
)
diff --git a/tools/go/go_mirrors.bzl b/tools/go/go_mirrors.bzl
index 8865d69..bfee0a9 100644
--- a/tools/go/go_mirrors.bzl
+++ b/tools/go/go_mirrors.bzl
@@ -147,13 +147,6 @@
"strip_prefix": "github.com/grpc-ecosystem/grpc-gateway@v1.16.0",
"version": "v1.16.0",
},
- "com_github_joho_godotenv": {
- "filename": "com_github_joho_godotenv__v1.4.0.zip",
- "importpath": "github.com/joho/godotenv",
- "sha256": "6c6d2f6c2a9d2ee8608e4acd7ba7035d31b1f0da3c7d9537a32928b8eed4e3cd",
- "strip_prefix": "github.com/joho/godotenv@v1.4.0",
- "version": "v1.4.0",
- },
"com_github_mattn_go_sqlite3": {
"filename": "com_github_mattn_go_sqlite3__v1.14.10.zip",
"importpath": "github.com/mattn/go-sqlite3",
diff --git a/tools/go/mirror_go_repos.py b/tools/go/mirror_go_repos.py
index 52e9e26..ae8f722 100644
--- a/tools/go/mirror_go_repos.py
+++ b/tools/go/mirror_go_repos.py
@@ -14,9 +14,6 @@
import sys
import tarfile
from typing import List, Dict
-import urllib.request
-
-from bazel_tools.tools.python.runfiles import runfiles
# Need a fully qualified import here because @bazel_tools interferes.
import org_frc971.tools.go.mirror_lib
@@ -101,7 +98,14 @@
def main(argv):
parser = argparse.ArgumentParser()
- parser.add_argument(
+ group = parser.add_mutually_exclusive_group()
+ group.add_argument(
+ "--prune",
+ action="store_true",
+ help=("When set, makes the tool prune go_mirrors_bzl to match the "
+ "repositories specified in go_deps_bzl. Incompatible with "
+ "--ssh_host."))
+ group.add_argument(
"--ssh_host",
type=str,
help=("The SSH host to copy the downloaded Go repositories to. This "
@@ -121,37 +125,38 @@
else:
existing_mirrored_repos = {}
- with tarfile.open("go_deps.tar", "w") as tar:
- cached_info = download_repos(repos, existing_mirrored_repos, tar)
- num_not_already_mirrored = len(tar.getnames())
+ exit_code = 0
- print(f"Found {num_not_already_mirrored}/{len(cached_info)} libraries "
- "that need to be mirrored.")
-
- # Only mirror the deps if we've specified an SSH host and we actually have
- # something to mirror.
- if args.ssh_host and num_not_already_mirrored:
- copy_to_host_and_unpack("go_deps.tar", args.ssh_host)
+ if args.prune:
+ # Delete all mirror info that is not needed anymore.
+ existing_cache_info = org_frc971.tools.go.mirror_lib.parse_go_mirror_info(args.go_mirrors_bzl)
+ cached_info = {}
+ for repo in repos:
+ try:
+ cached_info[repo["name"]] = existing_cache_info[repo["name"]]
+ except KeyError:
+ print(f"{repo['name']} needs to be mirrored still.")
+ exit_code = 1
else:
- print("Skipping mirroring because of lack of --ssh_host or there's "
- "nothing to actually mirror.")
+ # Download all the repositories that need to be mirrored.
+ with tarfile.open("go_deps.tar", "w") as tar:
+ cached_info = download_repos(repos, existing_mirrored_repos, tar)
+ num_not_already_mirrored = len(tar.getnames())
- with open(args.go_mirrors_bzl, "w") as file:
- file.write("# This file is auto-generated. Do not edit.\n")
- file.write("GO_MIRROR_INFO = ")
- # Format as JSON first. It's parsable as Starlark.
- json.dump(cached_info, file, indent=4, sort_keys=True)
- file.write("\n")
+ print(f"Found {num_not_already_mirrored}/{len(cached_info)} libraries "
+ "that need to be mirrored.")
+ # Only mirror the deps if we've specified an SSH host and we actually have
+ # something to mirror.
+ if args.ssh_host and num_not_already_mirrored:
+ copy_to_host_and_unpack("go_deps.tar", args.ssh_host)
+ else:
+ print("Skipping mirroring because of lack of --ssh_host or there's "
+ "nothing to actually mirror.")
- # Properly format the file now so that the linter doesn't complain.
- r = runfiles.Create()
- subprocess.run(
- [
- r.Rlocation("com_github_bazelbuild_buildtools/buildifier/buildifier_/buildifier"),
- args.go_mirrors_bzl,
- ],
- check=True)
+ org_frc971.tools.go.mirror_lib.write_go_mirror_info(args.go_mirrors_bzl, cached_info)
+
+ return exit_code
if __name__ == "__main__":
diff --git a/tools/go/mirror_lib.py b/tools/go/mirror_lib.py
index faad896..ea23abc 100644
--- a/tools/go/mirror_lib.py
+++ b/tools/go/mirror_lib.py
@@ -1,7 +1,11 @@
"""Provides helper functions for mirroring Go repositories."""
-import unittest.mock
+import json
+import subprocess
from typing import List, Dict
+import unittest.mock
+
+from bazel_tools.tools.python.runfiles import runfiles
def read_file(filepath: str) -> str:
@@ -39,3 +43,30 @@
return repositories
+
+def parse_go_mirror_info(filepath: str) -> Dict[str, Dict[str, str]]:
+ """Parses the tools/go/go_mirrors.bzl file and returns the GO_MIRROR_INFO dictionary."""
+ global_data = {}
+ compiled_code = compile(read_file(filepath), filepath, "exec")
+ eval(compiled_code, global_data)
+
+ return global_data["GO_MIRROR_INFO"]
+
+
+def write_go_mirror_info(filepath: str, mirror_info: Dict[str, Dict[str, str]]):
+ """Saves the specified mirror_info as GO_MIRROR_INFO into tools/go/go_mirrors.bzl."""
+ with open(filepath, "w") as file:
+ file.write("# This file is auto-generated. Do not edit.\n")
+ file.write("GO_MIRROR_INFO = ")
+ # Format as JSON first. It's parsable as Starlark.
+ json.dump(mirror_info, file, indent=4, sort_keys=True)
+ file.write("\n")
+
+ # Properly format the file now so that the linter doesn't complain.
+ r = runfiles.Create()
+ subprocess.run(
+ [
+ r.Rlocation("com_github_bazelbuild_buildtools/buildifier/buildifier_/buildifier"),
+ filepath,
+ ],
+ check=True)
diff --git a/tools/go/tweak_gazelle_go_deps.py b/tools/go/tweak_gazelle_go_deps.py
index 7189d43..d9d7901 100644
--- a/tools/go/tweak_gazelle_go_deps.py
+++ b/tools/go/tweak_gazelle_go_deps.py
@@ -12,14 +12,14 @@
import sys
import textwrap
-import tools.go.mirror_lib
+import org_frc971.tools.go.mirror_lib
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument("go_deps_bzl", type=str)
args = parser.parse_args(argv[1:])
- repos = tools.go.mirror_lib.parse_go_repositories(args.go_deps_bzl)
+ repos = org_frc971.tools.go.mirror_lib.parse_go_repositories(args.go_deps_bzl)
with open(args.go_deps_bzl, "w") as file:
file.write(textwrap.dedent("""\