Deploy the Julia runtime to the scouting server
This patch creates a .deb package to deploy Julia and the
DriverRank.jl dependencies to the scouting server.
This should let the deployed scouting server run the script without
too much trouble.
Signed-off-by: Philipp Schrader <philipp.schrader@gmail.com>
Change-Id: I419808b1567d5d56f2052864843b7f3b53223d31
diff --git a/WORKSPACE b/WORKSPACE
index 8a6206d..9bb5f38 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -1579,3 +1579,19 @@
strip_prefix = "tensorflow-bazel",
url = "https://www.frc971.org/Build-Dependencies/tensorflow-2.8.0.tar.gz",
)
+
+http_archive(
+ name = "julia",
+ build_file = "//third_party:julia/julia.BUILD",
+ patch_cmds = [
+ "echo 'LIB_SYMLINKS = {' > files.bzl",
+ '''find lib/ -type l -exec bash -c 'echo "\\"{}\\": \\"$(readlink {})\\","' \\; | sort >> files.bzl''',
+ "echo '}' >> files.bzl",
+ "echo 'LIBS = [' >> files.bzl",
+ '''find lib/ -type f -exec bash -c 'echo "\\"{}\\","' \\; | sort >> files.bzl''',
+ "echo ']' >> files.bzl",
+ ],
+ sha256 = "e71a24816e8fe9d5f4807664cbbb42738f5aa9fe05397d35c81d4c5d649b9d05",
+ strip_prefix = "julia-1.8.5",
+ url = "https://julialang-s3.julialang.org/bin/linux/x64/1.8/julia-1.8.5-linux-x86_64.tar.gz",
+)
diff --git a/scouting/DriverRank/BUILD b/scouting/DriverRank/BUILD
index e82fbfb..0e8c8d5 100644
--- a/scouting/DriverRank/BUILD
+++ b/scouting/DriverRank/BUILD
@@ -1,3 +1,5 @@
+load("@rules_pkg//:pkg.bzl", "pkg_deb", "pkg_tar")
+
filegroup(
name = "driver_rank_script",
srcs = [
@@ -5,3 +7,57 @@
],
visibility = ["//scouting:__subpackages__"],
)
+
+pkg_tar(
+ name = "julia_runtime",
+ package_dir = "opt/frc971/julia_runtime",
+ deps = [
+ "@julia//:runtime",
+ ],
+)
+
+pkg_tar(
+ name = "julia_manifest",
+ srcs = [
+ "Manifest.toml",
+ "Project.toml",
+ "activate.jl",
+ ],
+ package_dir = "opt/frc971/julia_manifest",
+)
+
+pkg_tar(
+ name = "julia_files",
+ deps = [
+ ":julia_manifest",
+ ":julia_runtime",
+ ],
+)
+
+pkg_deb(
+ name = "frc971-scouting-julia",
+ architecture = "amd64",
+ data = ":julia_files",
+ description = "The Julia files for the FRC971 scouting web server.",
+ maintainer = "frc971@frc971.org",
+ package = "frc971-scouting-julia",
+ postinst = "postinst",
+ version = "1",
+)
+
+py_binary(
+ name = "deploy",
+ srcs = [
+ "deploy.py",
+ ],
+ args = [
+ "--deb",
+ "$(location :frc971-scouting-julia)",
+ ],
+ data = [
+ ":frc971-scouting-julia",
+ ],
+ deps = [
+ "//scouting/deploy",
+ ],
+)
diff --git a/scouting/DriverRank/README.md b/scouting/DriverRank/README.md
new file mode 100644
index 0000000..b744755
--- /dev/null
+++ b/scouting/DriverRank/README.md
@@ -0,0 +1,15 @@
+# Driver ranking parsing script
+
+This directory contains the script that parses the raw data that the scouts
+collect on driver rankings and gives each team a score.
+
+## Deployment
+
+Whenever the Julia environment is set up for the first time or whenever the
+dependencies are updated, the Julia package needs to be redeployed. This is a
+separate step from the scouting server deployment because the Julia runtime is
+huge.
+
+```console
+$ bazel run //scouting/DriverRank:deploy -- --host scouting
+```
diff --git a/scouting/DriverRank/activate.jl b/scouting/DriverRank/activate.jl
new file mode 100644
index 0000000..741d05d
--- /dev/null
+++ b/scouting/DriverRank/activate.jl
@@ -0,0 +1,4 @@
+# A small helper to install all the dependencies on the scouting server.
+using Pkg
+Pkg.activate(ARGS[1])
+Pkg.instantiate()
diff --git a/scouting/DriverRank/deploy.py b/scouting/DriverRank/deploy.py
new file mode 100644
index 0000000..6e88edf
--- /dev/null
+++ b/scouting/DriverRank/deploy.py
@@ -0,0 +1,6 @@
+import sys
+
+from org_frc971.scouting.deploy.deploy import main
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv))
diff --git a/scouting/DriverRank/postinst b/scouting/DriverRank/postinst
new file mode 100644
index 0000000..3c21b2b
--- /dev/null
+++ b/scouting/DriverRank/postinst
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+export PATH="/opt/frc971/julia_runtime/bin:${PATH}"
+export JULIA_DEPOT_PATH=/var/frc971/scouting/julia_depot/
+export JULIA_PROJECT=/opt/frc971/julia_manifest
+
+julia /opt/frc971/julia_manifest/activate.jl /opt/frc971/julia_manifest
diff --git a/scouting/deploy/BUILD b/scouting/deploy/BUILD
index 2bf4b4e..eb8b537 100644
--- a/scouting/deploy/BUILD
+++ b/scouting/deploy/BUILD
@@ -42,6 +42,9 @@
name = "frc971-scouting-server",
architecture = "amd64",
data = ":deploy_tar",
+ depends = [
+ "frc971-scouting-julia",
+ ],
description = "The FRC971 scouting web server.",
# TODO(phil): What's a good email address for this?
maintainer = "frc971@frc971.org",
@@ -66,4 +69,5 @@
data = [
":frc971-scouting-server",
],
+ visibility = ["//scouting/DriverRank:__pkg__"],
)
diff --git a/scouting/deploy/scouting.service b/scouting/deploy/scouting.service
index 5aa64b0..94582cd 100644
--- a/scouting/deploy/scouting.service
+++ b/scouting/deploy/scouting.service
@@ -8,6 +8,11 @@
Type=simple
WorkingDirectory=/opt/frc971/scouting_server
Environment=RUNFILES_DIR=/opt/frc971/scouting_server
+# Add "julia" to the PATH.
+Environment=PATH=/opt/frc971/scouting/julia_runtime/bin:/usr/local/bin:/usr/bin:/bin
+# Use the Julia cache set up by the frc971-scouting-julia package.
+Environment=JULIA_DEPOT_PATH=/var/frc971/scouting/julia_depot/
+Environment=JULIA_PROJECT=/opt/frc971/julia_manifest
ExecStart=/opt/frc971/scouting_server/scouting/scouting \
-port 8080 \
-db_config /var/frc971/scouting/db_config.json \
diff --git a/third_party/julia/julia.BUILD b/third_party/julia/julia.BUILD
new file mode 100644
index 0000000..94988ca
--- /dev/null
+++ b/third_party/julia/julia.BUILD
@@ -0,0 +1,18 @@
+load("@rules_pkg//:pkg.bzl", "pkg_tar")
+load(":files.bzl", "LIB_SYMLINKS", "LIBS")
+
+pkg_tar(
+ name = "runtime",
+ srcs = LIBS + [
+ "bin/julia",
+ ] + glob([
+ "share/julia/**/*.jl",
+ "share/julia/**/*.toml",
+ "include/julia/**/*",
+ ], exclude = [
+ "**/test/**",
+ ]),
+ symlinks = LIB_SYMLINKS,
+ strip_prefix = "external/julia",
+ visibility = ["//visibility:public"],
+)
diff --git a/tools/dependency_rewrite b/tools/dependency_rewrite
index 7e9e1de..16fe927 100644
--- a/tools/dependency_rewrite
+++ b/tools/dependency_rewrite
@@ -9,6 +9,7 @@
rewrite static.rust-lang.org/(.*) software.frc971.org/Build-Dependencies/static.rust-lang.org/$1
rewrite storage.googleapis.com/(.*) software.frc971.org/Build-Dependencies/storage.googleapis.com/$1
rewrite files.pythonhosted.org/(.*) software.frc971.org/Build-Dependencies/files.pythonhosted.org/$1
+rewrite julialang-s3.julialang.org/(.*) software.frc971.org/Build-Dependencies/julialang-s3.julialang.org/$1
rewrite devsite.ctr-electronics.com/(.*) software.frc971.org/Build-Dependencies/devsite.ctr-electronics.com/$1
rewrite www.openssl.org/(.*) software.frc971.org/Build-Dependencies/www.openssl.org/$1
rewrite zlib.net/(.*) software.frc971.org/Build-Dependencies/zlib.net/$1