Create scraping library to get match data
To use it, you need to make a JSON config file which gets stored
at the workspace root or you pass the path into the function(s)
Change-Id: I9d074c7ad84c45b418eb5be8b316a84b022db835
Signed-off-by: Het Satasiya <satasiyahet@gmail.com>
diff --git a/.gitignore b/.gitignore
index 8c6aa64..8eeac66 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,10 @@
# default. We don't want folks to check it in.
/ldap.json
+# The scraping library uses looks for this config file by default,
+# you don't want to get that checked in
+/scouting_config.json
+
# Hide vagrant's files that unfortunately make it into the source tree when you
# run "vagrant up".
/vm/.vagrant/
diff --git a/go_deps.bzl b/go_deps.bzl
index 30d0311..a445eeb 100644
--- a/go_deps.bzl
+++ b/go_deps.bzl
@@ -129,6 +129,12 @@
version = "v1.16.0",
)
maybe_override_go_dep(
+ name = "com_github_joho_godotenv",
+ importpath = "github.com/joho/godotenv",
+ sum = "h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=",
+ version = "v1.4.0",
+ )
+ maybe_override_go_dep(
name = "com_github_mattn_go_sqlite3",
importpath = "github.com/mattn/go-sqlite3",
sum = "h1:MLn+5bFRlWMGoSRmJour3CL1w/qL96mvipqpwQW/Sfk=",
diff --git a/scouting/scraping/BUILD b/scouting/scraping/BUILD
new file mode 100644
index 0000000..58db2b1
--- /dev/null
+++ b/scouting/scraping/BUILD
@@ -0,0 +1,12 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+ name = "scraping",
+ srcs = [
+ "scrape.go",
+ "types.go",
+ ],
+ importpath = "github.com/frc971/971-Robot-Code/scouting/scraping",
+ target_compatible_with = ["@platforms//cpu:x86_64"],
+ visibility = ["//visibility:public"],
+)
diff --git a/scouting/scraping/scrape.go b/scouting/scraping/scrape.go
new file mode 100644
index 0000000..fa20f7b
--- /dev/null
+++ b/scouting/scraping/scrape.go
@@ -0,0 +1,86 @@
+package scraping
+
+// A library to grab match data from The Blue Alliance.
+import (
+ "encoding/json"
+ "errors"
+ "io/ioutil"
+ "log"
+ "net/http"
+ "os"
+)
+
+// Stores the TBA API key to access the API.
+type params struct {
+ ApiKey string `json:"api_key"`
+}
+
+// Takes in year and FIRST event code and returns all matches in that event according to TBA.
+// Also takes in a file path to the JSON config file that contains your TBA API key.
+// It defaults to <workspace root>/config.json
+// the config is expected to have the following contents:
+//{
+// api_key:"myTBAapiKey"
+//}
+func AllMatches(year, eventCode, filePath string) ([]Match, error) {
+ if filePath == "" {
+ filePath = os.Getenv("BUILD_WORKSPACE_DIRECTORY") + "/scouting_config.json"
+ }
+ // Takes the filepath and grabs the api key from the json.
+ content, err := ioutil.ReadFile(filePath)
+ if err != nil {
+ log.Fatal(err)
+ }
+ // Parses the JSON parameters into a struct.
+ var passed_params params
+ error := json.Unmarshal([]byte(content), &passed_params)
+ if error != nil {
+ log.Fatalf("You forgot to add the api_key parameter in the json file")
+ log.Fatalf("%s", err)
+ }
+
+ // Create the TBA event key for the year and event code.
+ eventKey := year + eventCode
+
+ // Create the client for HTTP requests.
+ client := &http.Client{}
+
+ // Create a get request for the match info.
+ req, err := http.NewRequest("GET", "https://www.thebluealliance.com/api/v3/event/"+eventKey+"/matches", nil)
+
+ if err != nil {
+ return nil, errors.New("failed to build http request")
+ }
+
+ // Add the auth key header to the request.
+ req.Header.Add("X-TBA-Auth-Key", passed_params.ApiKey)
+
+ // Make the API request
+ resp, err := client.Do(req)
+
+ if err != nil {
+ return nil, err
+ }
+
+ if resp.Status != "200 OK" {
+ return nil, errors.New("Recieved a status of " + resp.Status + " expected : 200 OK")
+ }
+
+ // Wait until the response is done.
+ defer resp.Body.Close()
+
+ // Get all bytes from response body.
+ bodyBytes, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return nil, errors.New("failed to read response body with error :" + err.Error())
+ }
+
+ var matches []Match
+ // Unmarshal json into go usable format.
+ jsonError := json.Unmarshal([]byte(bodyBytes), &matches)
+ if jsonError != nil {
+ return nil, errors.New("failed to unmarshal json recieved from TBA")
+ }
+
+ return matches, nil
+}
diff --git a/scouting/scraping/types.go b/scouting/scraping/types.go
new file mode 100644
index 0000000..aa0d4be
--- /dev/null
+++ b/scouting/scraping/types.go
@@ -0,0 +1,82 @@
+package scraping
+
+// Match holds the TBA data for a given match
+type Match struct {
+ Key string
+ CompLevel string `json:"comp_level"`
+ SetNumber int `json:"set_number"`
+ MatchNumber int `json:"match_number"`
+ Alliances Alliances `json:"alliances"`
+ WinningAlliance string `json:"winning_alliance"`
+ EventKey string `json:"event_key"`
+ Time int `json:"time"`
+ PredictedTime int `json:"predicted_time"`
+ ActualTime int `json:"actual_time"`
+ PostResultTime int `json:"post_result_time"`
+ ScoreBreakdowns ScoreBreakdowns `json:"score_breakdowns"`
+}
+
+// Holds score breakdowns for each alliance
+type ScoreBreakdowns struct {
+ Blue ScoreBreakdownAlliance `json:"blue"`
+ Red ScoreBreakdownAlliance `json:"red"`
+}
+
+// Stores the actual data for the breakdown
+type ScoreBreakdownAlliance struct {
+ TaxiRobot1 string `json:"taxiRobot1"`
+ EndgameRobot1 string `json:"endgameRobot1"`
+ TaxiRobot2 string `json:"taxiRobot2"`
+ EndgameRobot2 string `json:"endgameRobot2"`
+ TaxiRobot3 string `json:"taxiRobot3"`
+ EndgameRobot3 string `json:"endgameRobot3"`
+
+ AutoCargoLowerNear int `json:"autoCargoLowerNear"`
+ AutoCargoLowerFar int `json:"autoCargoLowerFar"`
+ AutoCargoLowerBlue int `json:"autoCargoLowerBlue"`
+ AutoCargoLowerRed int `json:"autoCargoLowerRed"`
+ AutoCargoUpperNear int `json:"autoCargoUpperNear"`
+ AutoCargoUpperFar int `json:"autoCargoUpperFar"`
+ AutoCargoUpperBlue int `json:"autoCargoUpperBlue"`
+ AutoCargoUpperRed int `json:"autoCargoUpperRed"`
+ AutoCargoTotal int `json:"autoCargoTotal"`
+ TeleOpCargoLowerNear int `json:"teleopCargoLowerNear"`
+ TeleOpCargoLowerFar int `json:"teleopCargoLowerFar"`
+ TeleOpCargoLowerBlue int `json:"teleopCargoLowerBlue"`
+ TeleOpCargoLowerRed int `json:"teleopCargoLowerRed"`
+ TeleOpCargoUpperNear int `json:"teleopCargoUpperNear"`
+ TeleOpCargoUpperFar int `json:"teleopCargoUpperFar"`
+ TeleOpCargoUpperBlue int `json:"teleopCargoUpperBlue"`
+ TeleOpCargoUpperRed int `json:"teleopCargoUpperRed"`
+ TeleopCargoTotal int `json:"teleopCargoTotal"`
+ MatchCargoTotal int `json:"matchCargoTotal"`
+ AutoTaxiPoints int `json:"autoTaxiPoints"`
+ AutoCargoPoints int `json:"autoCargoPoints"`
+ AutoPoints int `json:"autoPoints"`
+ QuintetAchieved bool `json:"quintetAchieved"`
+ TeleOpCargoPoints int `json:"teleopCargoPoints"`
+ EndgamePoints int `json:"endgamePoints"`
+ TeleOpPoints int `json:"teleopPoints"`
+ CargoBonusRankingPoint bool `json:"cargoBonusRankingPoint"`
+ HangerBonusRankingPoint bool `json:"hangarBonusRankingPoint"`
+ FoulCount bool `json:"foulCount"`
+ TechFoulCount int `json:"techFoulCount"`
+ AdjustPoints int `json:"adjustPoints"`
+ FoulPoints int `json:"foulPoints"`
+ RankingPoints int `json:"rp"`
+ TotalPoints int `json:"totalPoints"`
+}
+
+// Alliances holds the two alliances for a match
+type Alliances struct {
+ Red Alliance `json:"red"`
+ Blue Alliance `json:"blue"`
+}
+
+// Alliance holds the info for the alliance
+type Alliance struct {
+ Score int `json:"score"`
+ TeamKeys []string `json:"team_keys"`
+ SurrogateTeamKeys []string `json:"surrogate_team_keys"`
+ DqTeamKeys []string `json:"dq_team_keys"`
+}
diff --git a/tools/go/go_mirrors.bzl b/tools/go/go_mirrors.bzl
index 0e8f394..8865d69 100644
--- a/tools/go/go_mirrors.bzl
+++ b/tools/go/go_mirrors.bzl
@@ -147,6 +147,20 @@
"strip_prefix": "github.com/grpc-ecosystem/grpc-gateway@v1.16.0",
"version": "v1.16.0",
},
+ "com_github_joho_godotenv": {
+ "filename": "com_github_joho_godotenv__v1.4.0.zip",
+ "importpath": "github.com/joho/godotenv",
+ "sha256": "6c6d2f6c2a9d2ee8608e4acd7ba7035d31b1f0da3c7d9537a32928b8eed4e3cd",
+ "strip_prefix": "github.com/joho/godotenv@v1.4.0",
+ "version": "v1.4.0",
+ },
+ "com_github_mattn_go_sqlite3": {
+ "filename": "com_github_mattn_go_sqlite3__v1.14.10.zip",
+ "importpath": "github.com/mattn/go-sqlite3",
+ "sha256": "3c1e6497b023fc4741bf1bbfb39ae657b99cf44cfb33f5cbf1e88b315d25a306",
+ "strip_prefix": "github.com/mattn/go-sqlite3@v1.14.10",
+ "version": "v1.14.10",
+ },
"com_github_pmezard_go_difflib": {
"filename": "com_github_pmezard_go_difflib__v1.0.0.zip",
"importpath": "github.com/pmezard/go-difflib",
@@ -315,11 +329,4 @@
"strip_prefix": "golang.org/x/xerrors@v0.0.0-20200804184101-5ec99f83aff1",
"version": "v0.0.0-20200804184101-5ec99f83aff1",
},
- "com_github_mattn_go_sqlite3": {
- "filename": "com_github_mattn_go_sqlite3__v1.14.10.zip",
- "importpath": "github.com/mattn/go-sqlite3",
- "sha256": "3c1e6497b023fc4741bf1bbfb39ae657b99cf44cfb33f5cbf1e88b315d25a306",
- "strip_prefix": "github.com/mattn/go-sqlite3@v1.14.10",
- "version": "v1.14.10",
- },
}