scouting: Add an endpoint for populating the match schedule
This patch combines the scraping library with the scouting webserver.
There's now also a new end point for the web page (or debug CLI tool)
to ask the server to fetch the match list. The end point is
`/requests/refresh_match_list`.
All the tests are updated. The `cli_test` downloads a 2016 ny_tr match
list that I downloaded from TBA. It should be a decent integration
test as it uses representative data.
Signed-off-by: Philipp Schrader <philipp.schrader@gmail.com>
Change-Id: I6c540590521b00887eb2ddde2a9369875c659551
diff --git a/scouting/webserver/requests/debug/BUILD b/scouting/webserver/requests/debug/BUILD
index e3028dc..402503f 100644
--- a/scouting/webserver/requests/debug/BUILD
+++ b/scouting/webserver/requests/debug/BUILD
@@ -8,6 +8,7 @@
visibility = ["//visibility:public"],
deps = [
"//scouting/webserver/requests/messages:error_response_go_fbs",
+ "//scouting/webserver/requests/messages:refresh_match_list_response_go_fbs",
"//scouting/webserver/requests/messages:request_all_matches_response_go_fbs",
"//scouting/webserver/requests/messages:request_data_scouting_response_go_fbs",
"//scouting/webserver/requests/messages:request_matches_for_team_response_go_fbs",
diff --git a/scouting/webserver/requests/debug/cli/BUILD b/scouting/webserver/requests/debug/cli/BUILD
index aba9177..903f8c8 100644
--- a/scouting/webserver/requests/debug/cli/BUILD
+++ b/scouting/webserver/requests/debug/cli/BUILD
@@ -10,7 +10,10 @@
importpath = "github.com/frc971/971-Robot-Code/scouting/webserver/requests/debug/cli",
target_compatible_with = ["@platforms//cpu:x86_64"],
visibility = ["//visibility:private"],
- deps = ["//scouting/webserver/requests/debug"],
+ deps = [
+ "//scouting/webserver/requests/debug",
+ "@com_github_davecgh_go_spew//spew",
+ ],
)
go_binary(
@@ -27,6 +30,7 @@
],
data = [
":cli",
+ "//scouting/scraping:test_data",
"//scouting/webserver",
],
)
diff --git a/scouting/webserver/requests/debug/cli/cli_test.py b/scouting/webserver/requests/debug/cli/cli_test.py
index 347c828..a737d59 100644
--- a/scouting/webserver/requests/debug/cli/cli_test.py
+++ b/scouting/webserver/requests/debug/cli/cli_test.py
@@ -2,6 +2,7 @@
import json
import os
+import re
from pathlib import Path
import shutil
import socket
@@ -10,11 +11,10 @@
from typing import Any, Dict, List
import unittest
-def write_json(content: Dict[str, Any]):
+def write_json_request(content: Dict[str, Any]):
"""Writes a JSON file with the specified dict content."""
json_path = Path(os.environ["TEST_TMPDIR"]) / "test.json"
- with open(json_path, "w") as file:
- file.write(json.dumps(content))
+ json_path.write_text(json.dumps(content))
return json_path
def run_debug_cli(args: List[str]):
@@ -30,28 +30,81 @@
run_result.stderr.decode("utf-8"),
)
+def wait_for_server(port: int):
+ """Waits for the server at the specified port to respond to TCP connections."""
+ while True:
+ try:
+ connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ connection.connect(("localhost", port))
+ connection.close()
+ break
+ except ConnectionRefusedError:
+ connection.close()
+ time.sleep(0.01)
+
+
class TestDebugCli(unittest.TestCase):
def setUp(self):
- self.webserver = subprocess.Popen(["scouting/webserver/webserver_/webserver"])
+ tmpdir = Path(os.environ["TEST_TMPDIR"]) / "temp"
+ try:
+ shutil.rmtree(tmpdir)
+ except FileNotFoundError:
+ pass
+ os.mkdir(tmpdir)
- # Wait for the server to respond to requests.
- while True:
- try:
- connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- connection.connect(("localhost", 8080))
- connection.close()
- break
- except ConnectionRefusedError:
- connection.close()
- time.sleep(0.01)
+ # Copy the test data into place so that the final API call can be
+ # emulated.
+ tba_api_dir = tmpdir / "api" / "v3" / "event" / "1234event_key"
+ os.makedirs(tba_api_dir)
+ (tba_api_dir / "matches").write_text(
+ Path("scouting/scraping/test_data/2016_nytr.json").read_text()
+ )
+
+ # Create a fake TBA server to serve the static match list.
+ self.fake_tba_api = subprocess.Popen(
+ ["python3", "-m", "http.server", "7000"],
+ cwd=tmpdir,
+ )
+
+ # Configure the scouting webserver to scrape data from our fake TBA
+ # server.
+ scouting_config = tmpdir / "scouting_config.json"
+ scouting_config.write_text(json.dumps({
+ "api_key": "dummy_key_that_is_not_actually_used_in_this_test",
+ "base_url": "http://localhost:7000",
+ }))
+
+ # Run the scouting webserver.
+ self.webserver = subprocess.Popen([
+ "scouting/webserver/webserver_/webserver",
+ "-port=8080",
+ "-database=%s/database.db" % tmpdir,
+ "-tba_config=%s/scouting_config.json" % tmpdir,
+ ])
+
+ # Wait for the servers to be reachable.
+ wait_for_server(7000)
+ wait_for_server(8080)
def tearDown(self):
+ self.fake_tba_api.terminate()
self.webserver.terminate()
+ self.fake_tba_api.wait()
self.webserver.wait()
+ def refresh_match_list(self):
+ """Triggers the webserver to fetch the match list."""
+ json_path = write_json_request({
+ "year": 1234,
+ "event_code": "event_key",
+ })
+ exit_code, stdout, stderr = run_debug_cli(["-refreshMatchList", json_path])
+ self.assertEqual(exit_code, 0, stderr)
+ self.assertIn("(refresh_match_list_response.RefreshMatchListResponseT)", stdout)
+
def test_submit_data_scouting(self):
- json_path = write_json({
+ json_path = write_json_request({
"team": 971,
"match": 42,
"missed_shots_auto": 9971,
@@ -70,31 +123,37 @@
self.assertIn("/requests/submit/data_scouting returned 501 Not Implemented", stderr)
def test_request_all_matches(self):
- # RequestAllMatches has no fields.
- json_path = write_json({})
- exit_code, _stdout, stderr = run_debug_cli(["-requestAllMatches", json_path])
+ self.refresh_match_list()
- # TODO(phil): Actually add some matches here.
- self.assertEqual(exit_code, 0)
- self.assertIn("{MatchList:[]}", stderr)
+ # RequestAllMatches has no fields.
+ json_path = write_json_request({})
+ exit_code, stdout, stderr = run_debug_cli(["-requestAllMatches", json_path])
+
+ self.assertEqual(exit_code, 0, stderr)
+ self.assertIn("MatchList: ([]*request_all_matches_response.MatchT) (len=90 cap=90) {", stdout)
+ self.assertEqual(stdout.count("MatchNumber:"), 90)
def test_request_matches_for_team(self):
- json_path = write_json({
- "team": 971,
- })
- exit_code, _stdout, stderr = run_debug_cli(["-requestMatchesForTeam", json_path])
+ self.refresh_match_list()
- # TODO(phil): Actually add some matches here.
- self.assertEqual(exit_code, 0)
- self.assertIn("{MatchList:[]}", stderr)
+ json_path = write_json_request({
+ "team": 4856,
+ })
+ exit_code, stdout, stderr = run_debug_cli(["-requestMatchesForTeam", json_path])
+
+ # Team 4856 has 12 matches.
+ self.assertEqual(exit_code, 0, stderr)
+ self.assertIn("MatchList: ([]*request_matches_for_team_response.MatchT) (len=12 cap=12) {", stdout)
+ self.assertEqual(stdout.count("MatchNumber:"), 12)
+ self.assertEqual(len(re.findall(r": \(int32\) 4856[,\n]", stdout)), 12)
def test_request_data_scouting(self):
- json_path = write_json({})
- exit_code, _stdout, stderr = run_debug_cli(["-requestDataScouting", json_path])
+ json_path = write_json_request({})
+ exit_code, stdout, stderr = run_debug_cli(["-requestDataScouting", json_path])
# TODO(phil): Actually add data here before querying it.
- self.assertEqual(exit_code, 0)
- self.assertIn("{StatsList:[]}", stderr)
+ self.assertEqual(exit_code, 0, stderr)
+ self.assertIn("(request_data_scouting_response.RequestDataScoutingResponseT)", stdout)
if __name__ == "__main__":
unittest.main()
diff --git a/scouting/webserver/requests/debug/cli/main.go b/scouting/webserver/requests/debug/cli/main.go
index 0782d82..03032be 100644
--- a/scouting/webserver/requests/debug/cli/main.go
+++ b/scouting/webserver/requests/debug/cli/main.go
@@ -11,6 +11,7 @@
"os/exec"
"path/filepath"
+ "github.com/davecgh/go-spew/spew"
"github.com/frc971/971-Robot-Code/scouting/webserver/requests/debug"
)
@@ -76,6 +77,8 @@
"If specified, parse the file as a RequestMatchesForTeam JSON request.")
requestDataScoutingPtr := flag.String("requestDataScouting", "",
"If specified, parse the file as a RequestDataScouting JSON request.")
+ refreshMatchListPtr := flag.String("refreshMatchList", "",
+ "If specified, parse the file as a RefreshMatchList JSON request.")
flag.Parse()
// Handle the actual arguments.
@@ -88,7 +91,7 @@
if err != nil {
log.Fatal("Failed SubmitDataScouting: ", err)
}
- log.Printf("%+v", *response)
+ spew.Dump(*response)
}
if *requestAllMatchesPtr != "" {
log.Printf("Sending RequestAllMatches to %s", *addressPtr)
@@ -99,7 +102,7 @@
if err != nil {
log.Fatal("Failed RequestAllMatches: ", err)
}
- log.Printf("%+v", *response)
+ spew.Dump(*response)
}
if *requestMatchesForTeamPtr != "" {
log.Printf("Sending RequestMatchesForTeam to %s", *addressPtr)
@@ -110,7 +113,7 @@
if err != nil {
log.Fatal("Failed RequestMatchesForTeam: ", err)
}
- log.Printf("%+v", *response)
+ spew.Dump(*response)
}
if *requestDataScoutingPtr != "" {
log.Printf("Sending RequestDataScouting to %s", *addressPtr)
@@ -121,6 +124,17 @@
if err != nil {
log.Fatal("Failed RequestDataScouting: ", err)
}
- log.Printf("%+v", *response)
+ spew.Dump(*response)
+ }
+ if *refreshMatchListPtr != "" {
+ log.Printf("Sending RefreshMatchList to %s", *addressPtr)
+ binaryRequest := parseJson(
+ "scouting/webserver/requests/messages/refresh_match_list.fbs",
+ *refreshMatchListPtr)
+ response, err := debug.RefreshMatchList(*addressPtr, binaryRequest)
+ if err != nil {
+ log.Fatal("Failed RefreshMatchList: ", err)
+ }
+ spew.Dump(*response)
}
}
diff --git a/scouting/webserver/requests/debug/debug.go b/scouting/webserver/requests/debug/debug.go
index 6515e81..81be3d1 100644
--- a/scouting/webserver/requests/debug/debug.go
+++ b/scouting/webserver/requests/debug/debug.go
@@ -9,6 +9,7 @@
"net/http"
"github.com/frc971/971-Robot-Code/scouting/webserver/requests/messages/error_response"
+ "github.com/frc971/971-Robot-Code/scouting/webserver/requests/messages/refresh_match_list_response"
"github.com/frc971/971-Robot-Code/scouting/webserver/requests/messages/request_all_matches_response"
"github.com/frc971/971-Robot-Code/scouting/webserver/requests/messages/request_data_scouting_response"
"github.com/frc971/971-Robot-Code/scouting/webserver/requests/messages/request_matches_for_team_response"
@@ -20,6 +21,7 @@
type RequestAllMatchesResponseT = request_all_matches_response.RequestAllMatchesResponseT
type RequestMatchesForTeamResponseT = request_matches_for_team_response.RequestMatchesForTeamResponseT
type RequestDataScoutingResponseT = request_data_scouting_response.RequestDataScoutingResponseT
+type RefreshMatchListResponseT = refresh_match_list_response.RefreshMatchListResponseT
// A struct that can be used as an `error`. It contains information about the
// why the server was unhappy and what the corresponding request was.
@@ -127,3 +129,15 @@
response := request_data_scouting_response.GetRootAsRequestDataScoutingResponse(responseBytes, 0)
return response.UnPack(), nil
}
+
+// Sends a `RefreshMatchList` message to the server and returns the
+// deserialized response.
+func RefreshMatchList(server string, requestBytes []byte) (*RefreshMatchListResponseT, error) {
+ responseBytes, err := performPost(server+"/requests/refresh_match_list", requestBytes)
+ if err != nil {
+ return nil, err
+ }
+ log.Printf("Parsing RefreshMatchListResponse")
+ response := refresh_match_list_response.GetRootAsRefreshMatchListResponse(responseBytes, 0)
+ return response.UnPack(), nil
+}