Merge changes I419808b1,Iaeeb0231,Iec2c0dcf,I6540f960
* changes:
Deploy the Julia runtime to the scouting server
Parametrize the DriverRank.jl script
Upgrade rules_pkg
Deploy the DriverRank.jl script as part of the scouting app
diff --git a/WORKSPACE b/WORKSPACE
index dd184d5..9bb5f38 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -99,14 +99,10 @@
http_archive(
name = "rules_pkg",
- patch_args = ["-p1"],
- patches = [
- "//third_party:rules_pkg/0001-Fix-tree-artifacts.patch",
- ],
- sha256 = "62eeb544ff1ef41d786e329e1536c1d541bb9bcad27ae984d57f18f314018e66",
+ sha256 = "8c20f74bca25d2d442b327ae26768c02cf3c99e93fad0381f32be9aab1967675",
urls = [
- "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.6.0/rules_pkg-0.6.0.tar.gz",
- "https://github.com/bazelbuild/rules_pkg/releases/download/0.6.0/rules_pkg-0.6.0.tar.gz",
+ "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.8.1/rules_pkg-0.8.1.tar.gz",
+ "https://github.com/bazelbuild/rules_pkg/releases/download/0.8.1/rules_pkg-0.8.1.tar.gz",
],
)
@@ -1583,3 +1579,19 @@
strip_prefix = "tensorflow-bazel",
url = "https://www.frc971.org/Build-Dependencies/tensorflow-2.8.0.tar.gz",
)
+
+http_archive(
+ name = "julia",
+ build_file = "//third_party:julia/julia.BUILD",
+ patch_cmds = [
+ "echo 'LIB_SYMLINKS = {' > files.bzl",
+ '''find lib/ -type l -exec bash -c 'echo "\\"{}\\": \\"$(readlink {})\\","' \\; | sort >> files.bzl''',
+ "echo '}' >> files.bzl",
+ "echo 'LIBS = [' >> files.bzl",
+ '''find lib/ -type f -exec bash -c 'echo "\\"{}\\","' \\; | sort >> files.bzl''',
+ "echo ']' >> files.bzl",
+ ],
+ sha256 = "e71a24816e8fe9d5f4807664cbbb42738f5aa9fe05397d35c81d4c5d649b9d05",
+ strip_prefix = "julia-1.8.5",
+ url = "https://julialang-s3.julialang.org/bin/linux/x64/1.8/julia-1.8.5-linux-x86_64.tar.gz",
+)
diff --git a/scouting/DriverRank/BUILD b/scouting/DriverRank/BUILD
index e82fbfb..0e8c8d5 100644
--- a/scouting/DriverRank/BUILD
+++ b/scouting/DriverRank/BUILD
@@ -1,3 +1,5 @@
+load("@rules_pkg//:pkg.bzl", "pkg_deb", "pkg_tar")
+
filegroup(
name = "driver_rank_script",
srcs = [
@@ -5,3 +7,57 @@
],
visibility = ["//scouting:__subpackages__"],
)
+
+pkg_tar(
+ name = "julia_runtime",
+ package_dir = "opt/frc971/julia_runtime",
+ deps = [
+ "@julia//:runtime",
+ ],
+)
+
+pkg_tar(
+ name = "julia_manifest",
+ srcs = [
+ "Manifest.toml",
+ "Project.toml",
+ "activate.jl",
+ ],
+ package_dir = "opt/frc971/julia_manifest",
+)
+
+pkg_tar(
+ name = "julia_files",
+ deps = [
+ ":julia_manifest",
+ ":julia_runtime",
+ ],
+)
+
+pkg_deb(
+ name = "frc971-scouting-julia",
+ architecture = "amd64",
+ data = ":julia_files",
+ description = "The Julia files for the FRC971 scouting web server.",
+ maintainer = "frc971@frc971.org",
+ package = "frc971-scouting-julia",
+ postinst = "postinst",
+ version = "1",
+)
+
+py_binary(
+ name = "deploy",
+ srcs = [
+ "deploy.py",
+ ],
+ args = [
+ "--deb",
+ "$(location :frc971-scouting-julia)",
+ ],
+ data = [
+ ":frc971-scouting-julia",
+ ],
+ deps = [
+ "//scouting/deploy",
+ ],
+)
diff --git a/scouting/DriverRank/README.md b/scouting/DriverRank/README.md
new file mode 100644
index 0000000..b744755
--- /dev/null
+++ b/scouting/DriverRank/README.md
@@ -0,0 +1,15 @@
+# Driver ranking parsing script
+
+This directory contains the script that parses the raw data that the scouts
+collect on driver rankings and gives each team a score.
+
+## Deployment
+
+Whenever the Julia environment is set up for the first time or whenever the
+dependencies are updated, the Julia package needs to be redeployed. This is a
+separate step from the scouting server deployment because the Julia runtime is
+huge.
+
+```console
+$ bazel run //scouting/DriverRank:deploy -- --host scouting
+```
diff --git a/scouting/DriverRank/activate.jl b/scouting/DriverRank/activate.jl
new file mode 100644
index 0000000..741d05d
--- /dev/null
+++ b/scouting/DriverRank/activate.jl
@@ -0,0 +1,4 @@
+# A small helper to install all the dependencies on the scouting server.
+using Pkg
+Pkg.activate(ARGS[1])
+Pkg.instantiate()
diff --git a/scouting/DriverRank/deploy.py b/scouting/DriverRank/deploy.py
new file mode 100644
index 0000000..6e88edf
--- /dev/null
+++ b/scouting/DriverRank/deploy.py
@@ -0,0 +1,6 @@
+import sys
+
+from org_frc971.scouting.deploy.deploy import main
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv))
diff --git a/scouting/DriverRank/postinst b/scouting/DriverRank/postinst
new file mode 100644
index 0000000..3c21b2b
--- /dev/null
+++ b/scouting/DriverRank/postinst
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+export PATH="/opt/frc971/julia_runtime/bin:${PATH}"
+export JULIA_DEPOT_PATH=/var/frc971/scouting/julia_depot/
+export JULIA_PROJECT=/opt/frc971/julia_manifest
+
+julia /opt/frc971/julia_manifest/activate.jl /opt/frc971/julia_manifest
diff --git a/scouting/DriverRank/src/DriverRank.jl b/scouting/DriverRank/src/DriverRank.jl
old mode 100644
new mode 100755
index c99be1f..39ac95e
--- a/scouting/DriverRank/src/DriverRank.jl
+++ b/scouting/DriverRank/src/DriverRank.jl
@@ -1,3 +1,5 @@
+#!/usr/bin/env julia
+
module DriverRank
using CSV
@@ -100,9 +102,11 @@
return result
end
-function rank()
- # TODO(phil): Make the input path configurable.
- df = DataFrame(CSV.File("./data/2022_madtown.csv"))
+function rank(
+ input_csv::String,
+ output_csv::String,
+)
+ df = DataFrame(CSV.File(input_csv))
rank1 = "Rank 1 (best)"
rank2 = "Rank 2"
@@ -133,10 +137,18 @@
:score=>Optim.minimizer(res),
) |>
x -> sort!(x, [:score], rev=true)
- # TODO(phil): Save the output to a CSV file.
- show(ranking_points, allrows=true)
+
+ # Uncomment to print the results on the console as well.
+ #show(ranking_points, allrows=true)
+
+ CSV.write(output_csv, ranking_points)
end
export rank
+# Run the program if this script is being executed from the command line.
+if abspath(PROGRAM_FILE) == @__FILE__
+ rank(ARGS[1], ARGS[2])
+end
+
end # module
diff --git a/scouting/deploy/BUILD b/scouting/deploy/BUILD
index 2bf4b4e..eb8b537 100644
--- a/scouting/deploy/BUILD
+++ b/scouting/deploy/BUILD
@@ -42,6 +42,9 @@
name = "frc971-scouting-server",
architecture = "amd64",
data = ":deploy_tar",
+ depends = [
+ "frc971-scouting-julia",
+ ],
description = "The FRC971 scouting web server.",
# TODO(phil): What's a good email address for this?
maintainer = "frc971@frc971.org",
@@ -66,4 +69,5 @@
data = [
":frc971-scouting-server",
],
+ visibility = ["//scouting/DriverRank:__pkg__"],
)
diff --git a/scouting/deploy/scouting.service b/scouting/deploy/scouting.service
index 5aa64b0..94582cd 100644
--- a/scouting/deploy/scouting.service
+++ b/scouting/deploy/scouting.service
@@ -8,6 +8,11 @@
Type=simple
WorkingDirectory=/opt/frc971/scouting_server
Environment=RUNFILES_DIR=/opt/frc971/scouting_server
+# Add "julia" to the PATH.
+Environment=PATH=/opt/frc971/scouting/julia_runtime/bin:/usr/local/bin:/usr/bin:/bin
+# Use the Julia cache set up by the frc971-scouting-julia package.
+Environment=JULIA_DEPOT_PATH=/var/frc971/scouting/julia_depot/
+Environment=JULIA_PROJECT=/opt/frc971/julia_manifest
ExecStart=/opt/frc971/scouting_server/scouting/scouting \
-port 8080 \
-db_config /var/frc971/scouting/db_config.json \
diff --git a/scouting/webserver/BUILD b/scouting/webserver/BUILD
index 934aff1..934b50a 100644
--- a/scouting/webserver/BUILD
+++ b/scouting/webserver/BUILD
@@ -9,6 +9,7 @@
deps = [
"//scouting/db",
"//scouting/scraping/background",
+ "//scouting/webserver/driver_ranking",
"//scouting/webserver/match_list",
"//scouting/webserver/rankings",
"//scouting/webserver/requests",
diff --git a/scouting/webserver/driver_ranking/driver_ranking.go b/scouting/webserver/driver_ranking/driver_ranking.go
index 005078f..82e34e1 100644
--- a/scouting/webserver/driver_ranking/driver_ranking.go
+++ b/scouting/webserver/driver_ranking/driver_ranking.go
@@ -119,7 +119,8 @@
// database.
outputRecords, err := readFromCsv(outputCsvFile)
- for _, record := range outputRecords {
+ // Skip the first row since those are the column labels.
+ for _, record := range outputRecords[1:] {
score, err := strconv.ParseFloat(record[1], 32)
if err != nil {
log.Println("Failed to parse score for team ", record[0], ": ", record[1], ": ", err)
diff --git a/scouting/webserver/driver_ranking/fake_driver_rank_script.py b/scouting/webserver/driver_ranking/fake_driver_rank_script.py
index 8f72267..c9a96eb 100644
--- a/scouting/webserver/driver_ranking/fake_driver_rank_script.py
+++ b/scouting/webserver/driver_ranking/fake_driver_rank_script.py
@@ -18,6 +18,7 @@
]
OUTPUT = [
+ ("team", "score"),
("1234", "1.5"),
("1235", "2.75"),
("1236", "4.0"),
diff --git a/scouting/webserver/main.go b/scouting/webserver/main.go
index 1811b7f..4752cf4 100644
--- a/scouting/webserver/main.go
+++ b/scouting/webserver/main.go
@@ -15,6 +15,7 @@
"github.com/frc971/971-Robot-Code/scouting/db"
"github.com/frc971/971-Robot-Code/scouting/scraping/background"
+ "github.com/frc971/971-Robot-Code/scouting/webserver/driver_ranking"
"github.com/frc971/971-Robot-Code/scouting/webserver/match_list"
"github.com/frc971/971-Robot-Code/scouting/webserver/rankings"
"github.com/frc971/971-Robot-Code/scouting/webserver/requests"
@@ -139,6 +140,13 @@
rankings.GetRankings(database, 0, "", *blueAllianceConfigPtr)
})
+ driverRankingParser := background.BackgroundScraper{}
+ driverRankingParser.Start(func() {
+ // Specify "" as the script path here so that the default is
+ // used.
+ driver_ranking.GenerateFullDriverRanking(database, "")
+ })
+
// Block until the user hits Ctrl-C.
sigint := make(chan os.Signal, 1)
signal.Notify(sigint, syscall.SIGINT)
@@ -149,6 +157,7 @@
fmt.Println("Shutting down.")
scoutingServer.Stop()
rankingsScraper.Stop()
+ driverRankingParser.Stop()
matchListScraper.Stop()
fmt.Println("Successfully shut down.")
}
diff --git a/third_party/julia/julia.BUILD b/third_party/julia/julia.BUILD
new file mode 100644
index 0000000..94988ca
--- /dev/null
+++ b/third_party/julia/julia.BUILD
@@ -0,0 +1,18 @@
+load("@rules_pkg//:pkg.bzl", "pkg_tar")
+load(":files.bzl", "LIB_SYMLINKS", "LIBS")
+
+pkg_tar(
+ name = "runtime",
+ srcs = LIBS + [
+ "bin/julia",
+ ] + glob([
+ "share/julia/**/*.jl",
+ "share/julia/**/*.toml",
+ "include/julia/**/*",
+ ], exclude = [
+ "**/test/**",
+ ]),
+ symlinks = LIB_SYMLINKS,
+ strip_prefix = "external/julia",
+ visibility = ["//visibility:public"],
+)
diff --git a/third_party/rules_pkg/0001-Fix-tree-artifacts.patch b/third_party/rules_pkg/0001-Fix-tree-artifacts.patch
deleted file mode 100644
index 567aba7..0000000
--- a/third_party/rules_pkg/0001-Fix-tree-artifacts.patch
+++ /dev/null
@@ -1,28 +0,0 @@
-From d654cc64ae71366ea82ac492106e9b2c8fa532d5 Mon Sep 17 00:00:00 2001
-From: Philipp Schrader <philipp.schrader@gmail.com>
-Date: Thu, 10 Mar 2022 23:25:21 -0800
-Subject: [PATCH] Fix tree artifacts
-
-For some reason the upstream code strips the directory names from the
-`babel()` rule that we use. This patch makes it so the directory is
-not stripped. This makes runfiles layout in the tarball match the
-runfiles layout in `bazel-bin`.
----
- pkg/pkg.bzl | 4 ++--
- 1 file changed, 2 insertions(+), 2 deletions(-)
-
-diff --git a/pkg/pkg.bzl b/pkg/pkg.bzl
-index d7adbbc..a241b26 100644
---- a/pkg/pkg.bzl
-+++ b/pkg/pkg.bzl
-@@ -157,8 +157,8 @@ def _pkg_tar_impl(ctx):
- # Tree artifacts need a name, but the name is never really
- # the important part. The likely behavior people want is
- # just the content, so we strip the directory name.
-- dest = "/".join(d_path.split("/")[0:-1])
-- add_tree_artifact(content_map, dest, f, src.label)
-+ #dest = "/".join(d_path.split("/")[0:-1])
-+ add_tree_artifact(content_map, d_path, f, src.label)
- else:
- # Note: This extra remap is the bottleneck preventing this
- # large block from being a utility method as shown below.
diff --git a/tools/dependency_rewrite b/tools/dependency_rewrite
index 7e9e1de..16fe927 100644
--- a/tools/dependency_rewrite
+++ b/tools/dependency_rewrite
@@ -9,6 +9,7 @@
rewrite static.rust-lang.org/(.*) software.frc971.org/Build-Dependencies/static.rust-lang.org/$1
rewrite storage.googleapis.com/(.*) software.frc971.org/Build-Dependencies/storage.googleapis.com/$1
rewrite files.pythonhosted.org/(.*) software.frc971.org/Build-Dependencies/files.pythonhosted.org/$1
+rewrite julialang-s3.julialang.org/(.*) software.frc971.org/Build-Dependencies/julialang-s3.julialang.org/$1
rewrite devsite.ctr-electronics.com/(.*) software.frc971.org/Build-Dependencies/devsite.ctr-electronics.com/$1
rewrite www.openssl.org/(.*) software.frc971.org/Build-Dependencies/www.openssl.org/$1
rewrite zlib.net/(.*) software.frc971.org/Build-Dependencies/zlib.net/$1