scouting: Scrape rankings and add to db
Signed-off-by: Yash Chainani <yashchainani28@gmail.com>
Change-Id: I9e84feeaaee640e98ba72919b022379de39f9d29
diff --git a/scouting/scraping/BUILD b/scouting/scraping/BUILD
index c643192..ed21433 100644
--- a/scouting/scraping/BUILD
+++ b/scouting/scraping/BUILD
@@ -6,6 +6,8 @@
# Generated with: bazel run //scouting/scraping:scraping_demo -- --json
"test_data/2016_nytr.json",
"test_data/2020_fake.json",
+ # Generated with: bazel run scouting/scraping:scraping_demo -- -category rankings -json >scouting/scraping/test_data/2016_nytr_rankings.json
+ "test_data/2016_nytr_rankings.json",
],
visibility = ["//visibility:public"],
)
diff --git a/scouting/scraping/test_data/2016_nytr_rankings.json b/scouting/scraping/test_data/2016_nytr_rankings.json
new file mode 100644
index 0000000..dc7ba1a
--- /dev/null
+++ b/scouting/scraping/test_data/2016_nytr_rankings.json
@@ -0,0 +1,796 @@
+{
+ "rankings": [
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 3.25
+ ],
+ "sort_orders": [
+ 39,
+ 310,
+ 165,
+ 448,
+ 600
+ ],
+ "record": {
+ "losses": 1,
+ "wins": 11,
+ "ties": 0
+ },
+ "rank": 1,
+ "dq": 0,
+ "team_key": "frc359"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 3.0833333333333335
+ ],
+ "sort_orders": [
+ 37,
+ 288,
+ 145,
+ 298,
+ 615
+ ],
+ "record": {
+ "losses": 1,
+ "wins": 11,
+ "ties": 0
+ },
+ "rank": 2,
+ "dq": 0,
+ "team_key": "frc5254"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 2.8333333333333335
+ ],
+ "sort_orders": [
+ 34,
+ 252,
+ 205,
+ 311,
+ 590
+ ],
+ "record": {
+ "losses": 1,
+ "wins": 11,
+ "ties": 0
+ },
+ "rank": 3,
+ "dq": 0,
+ "team_key": "frc3990"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 2.5833333333333335
+ ],
+ "sort_orders": [
+ 31,
+ 180,
+ 145,
+ 236,
+ 550
+ ],
+ "record": {
+ "losses": 4,
+ "wins": 8,
+ "ties": 0
+ },
+ "rank": 4,
+ "dq": 0,
+ "team_key": "frc5236"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 2.5
+ ],
+ "sort_orders": [
+ 30,
+ 304,
+ 140,
+ 284,
+ 615
+ ],
+ "record": {
+ "losses": 4,
+ "wins": 8,
+ "ties": 0
+ },
+ "rank": 5,
+ "dq": 0,
+ "team_key": "frc3419"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 2.3333333333333335
+ ],
+ "sort_orders": [
+ 28,
+ 260,
+ 150,
+ 191,
+ 575
+ ],
+ "record": {
+ "losses": 3,
+ "wins": 9,
+ "ties": 0
+ },
+ "rank": 6,
+ "dq": 0,
+ "team_key": "frc5240"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 2.3333333333333335
+ ],
+ "sort_orders": [
+ 28,
+ 254,
+ 125,
+ 208,
+ 595
+ ],
+ "record": {
+ "losses": 4,
+ "wins": 8,
+ "ties": 0
+ },
+ "rank": 7,
+ "dq": 0,
+ "team_key": "frc20"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 2.0833333333333335
+ ],
+ "sort_orders": [
+ 25,
+ 266,
+ 125,
+ 272,
+ 575
+ ],
+ "record": {
+ "losses": 5,
+ "wins": 7,
+ "ties": 0
+ },
+ "rank": 8,
+ "dq": 0,
+ "team_key": "frc48"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 2.0833333333333335
+ ],
+ "sort_orders": [
+ 25,
+ 260,
+ 140,
+ 155,
+ 550
+ ],
+ "record": {
+ "losses": 4,
+ "wins": 8,
+ "ties": 0
+ },
+ "rank": 9,
+ "dq": 0,
+ "team_key": "frc250"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 2
+ ],
+ "sort_orders": [
+ 24,
+ 200,
+ 145,
+ 227,
+ 560
+ ],
+ "record": {
+ "losses": 5,
+ "wins": 7,
+ "ties": 0
+ },
+ "rank": 10,
+ "dq": 0,
+ "team_key": "frc2791"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 2
+ ],
+ "sort_orders": [
+ 24,
+ 166,
+ 155,
+ 119,
+ 510
+ ],
+ "record": {
+ "losses": 5,
+ "wins": 7,
+ "ties": 0
+ },
+ "rank": 11,
+ "dq": 0,
+ "team_key": "frc358"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 1.9166666666666667
+ ],
+ "sort_orders": [
+ 23,
+ 278,
+ 120,
+ 160,
+ 585
+ ],
+ "record": {
+ "losses": 5,
+ "wins": 7,
+ "ties": 0
+ },
+ "rank": 12,
+ "dq": 0,
+ "team_key": "frc4930"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 1.9166666666666667
+ ],
+ "sort_orders": [
+ 23,
+ 268,
+ 120,
+ 131,
+ 655
+ ],
+ "record": {
+ "losses": 6,
+ "wins": 6,
+ "ties": 0
+ },
+ "rank": 13,
+ "dq": 0,
+ "team_key": "frc3044"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 1.9166666666666667
+ ],
+ "sort_orders": [
+ 23,
+ 262,
+ 120,
+ 176,
+ 550
+ ],
+ "record": {
+ "losses": 5,
+ "wins": 7,
+ "ties": 0
+ },
+ "rank": 14,
+ "dq": 0,
+ "team_key": "frc527"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 1.8333333333333333
+ ],
+ "sort_orders": [
+ 22,
+ 192,
+ 140,
+ 212,
+ 515
+ ],
+ "record": {
+ "losses": 6,
+ "wins": 6,
+ "ties": 0
+ },
+ "rank": 15,
+ "dq": 0,
+ "team_key": "frc3003"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 1.75
+ ],
+ "sort_orders": [
+ 21,
+ 300,
+ 120,
+ 275,
+ 560
+ ],
+ "record": {
+ "losses": 6,
+ "wins": 6,
+ "ties": 0
+ },
+ "rank": 16,
+ "dq": 0,
+ "team_key": "frc333"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 1.6666666666666667
+ ],
+ "sort_orders": [
+ 20,
+ 196,
+ 160,
+ 144,
+ 530
+ ],
+ "record": {
+ "losses": 7,
+ "wins": 5,
+ "ties": 0
+ },
+ "rank": 17,
+ "dq": 0,
+ "team_key": "frc1551"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 1.6666666666666667
+ ],
+ "sort_orders": [
+ 20,
+ 192,
+ 105,
+ 146,
+ 525
+ ],
+ "record": {
+ "losses": 6,
+ "wins": 6,
+ "ties": 0
+ },
+ "rank": 18,
+ "dq": 0,
+ "team_key": "frc1665"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 1.6666666666666667
+ ],
+ "sort_orders": [
+ 20,
+ 166,
+ 125,
+ 187,
+ 555
+ ],
+ "record": {
+ "losses": 8,
+ "wins": 4,
+ "ties": 0
+ },
+ "rank": 19,
+ "dq": 0,
+ "team_key": "frc663"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 1.6666666666666667
+ ],
+ "sort_orders": [
+ 20,
+ 156,
+ 130,
+ 119,
+ 525
+ ],
+ "record": {
+ "losses": 6,
+ "wins": 6,
+ "ties": 0
+ },
+ "rank": 20,
+ "dq": 0,
+ "team_key": "frc229"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 1.5833333333333333
+ ],
+ "sort_orders": [
+ 19,
+ 316,
+ 115,
+ 167,
+ 570
+ ],
+ "record": {
+ "losses": 7,
+ "wins": 5,
+ "ties": 0
+ },
+ "rank": 21,
+ "dq": 0,
+ "team_key": "frc4093"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 1.5833333333333333
+ ],
+ "sort_orders": [
+ 19,
+ 236,
+ 155,
+ 148,
+ 575
+ ],
+ "record": {
+ "losses": 7,
+ "wins": 5,
+ "ties": 0
+ },
+ "rank": 22,
+ "dq": 0,
+ "team_key": "frc1493"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 1.5833333333333333
+ ],
+ "sort_orders": [
+ 19,
+ 218,
+ 110,
+ 159,
+ 520
+ ],
+ "record": {
+ "losses": 6,
+ "wins": 6,
+ "ties": 0
+ },
+ "rank": 23,
+ "dq": 0,
+ "team_key": "frc5964"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 1.5833333333333333
+ ],
+ "sort_orders": [
+ 19,
+ 214,
+ 130,
+ 246,
+ 485
+ ],
+ "record": {
+ "losses": 6,
+ "wins": 6,
+ "ties": 0
+ },
+ "rank": 24,
+ "dq": 0,
+ "team_key": "frc145"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 1.4166666666666667
+ ],
+ "sort_orders": [
+ 17,
+ 184,
+ 110,
+ 122,
+ 440
+ ],
+ "record": {
+ "losses": 6,
+ "wins": 6,
+ "ties": 0
+ },
+ "rank": 25,
+ "dq": 0,
+ "team_key": "frc371"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 1.3333333333333333
+ ],
+ "sort_orders": [
+ 16,
+ 224,
+ 145,
+ 160,
+ 550
+ ],
+ "record": {
+ "losses": 8,
+ "wins": 4,
+ "ties": 0
+ },
+ "rank": 26,
+ "dq": 0,
+ "team_key": "frc5881"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 1.3333333333333333
+ ],
+ "sort_orders": [
+ 16,
+ 188,
+ 105,
+ 170,
+ 385
+ ],
+ "record": {
+ "losses": 4,
+ "wins": 5,
+ "ties": 0
+ },
+ "rank": 27,
+ "dq": 0,
+ "team_key": "frc3624"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 1.25
+ ],
+ "sort_orders": [
+ 15,
+ 224,
+ 95,
+ 141,
+ 525
+ ],
+ "record": {
+ "losses": 8,
+ "wins": 4,
+ "ties": 0
+ },
+ "rank": 28,
+ "dq": 0,
+ "team_key": "frc4508"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 1.25
+ ],
+ "sort_orders": [
+ 15,
+ 224,
+ 75,
+ 113,
+ 515
+ ],
+ "record": {
+ "losses": 7,
+ "wins": 5,
+ "ties": 0
+ },
+ "rank": 29,
+ "dq": 0,
+ "team_key": "frc5585"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 1.25
+ ],
+ "sort_orders": [
+ 15,
+ 214,
+ 130,
+ 203,
+ 505
+ ],
+ "record": {
+ "losses": 8,
+ "wins": 4,
+ "ties": 0
+ },
+ "rank": 30,
+ "dq": 0,
+ "team_key": "frc4481"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 1.25
+ ],
+ "sort_orders": [
+ 15,
+ 156,
+ 115,
+ 101,
+ 485
+ ],
+ "record": {
+ "losses": 7,
+ "wins": 4,
+ "ties": 0
+ },
+ "rank": 31,
+ "dq": 0,
+ "team_key": "frc4856"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 1.0833333333333333
+ ],
+ "sort_orders": [
+ 13,
+ 204,
+ 110,
+ 92,
+ 480
+ ],
+ "record": {
+ "losses": 7,
+ "wins": 4,
+ "ties": 0
+ },
+ "rank": 32,
+ "dq": 0,
+ "team_key": "frc5879"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 0.9166666666666666
+ ],
+ "sort_orders": [
+ 11,
+ 228,
+ 110,
+ 115,
+ 500
+ ],
+ "record": {
+ "losses": 9,
+ "wins": 3,
+ "ties": 0
+ },
+ "rank": 33,
+ "dq": 0,
+ "team_key": "frc1450"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 0.8333333333333334
+ ],
+ "sort_orders": [
+ 10,
+ 160,
+ 90,
+ 131,
+ 490
+ ],
+ "record": {
+ "losses": 10,
+ "wins": 2,
+ "ties": 0
+ },
+ "rank": 34,
+ "dq": 0,
+ "team_key": "frc5943"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 0.6666666666666666
+ ],
+ "sort_orders": [
+ 8,
+ 162,
+ 60,
+ 121,
+ 430
+ ],
+ "record": {
+ "losses": 10,
+ "wins": 2,
+ "ties": 0
+ },
+ "rank": 35,
+ "dq": 0,
+ "team_key": "frc4203"
+ },
+ {
+ "matches_played": 12,
+ "qual_average": 0,
+ "extra_stats": [
+ 0.6666666666666666
+ ],
+ "sort_orders": [
+ 8,
+ 144,
+ 65,
+ 140,
+ 430
+ ],
+ "record": {
+ "losses": 10,
+ "wins": 2,
+ "ties": 0
+ },
+ "rank": 36,
+ "dq": 0,
+ "team_key": "frc5149"
+ }
+ ]
+}
diff --git a/scouting/webserver/rankings/BUILD b/scouting/webserver/rankings/BUILD
new file mode 100644
index 0000000..c74f88f
--- /dev/null
+++ b/scouting/webserver/rankings/BUILD
@@ -0,0 +1,26 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+ name = "rankings",
+ srcs = ["rankings.go"],
+ importpath = "github.com/frc971/971-Robot-Code/scouting/webserver/rankings",
+ visibility = ["//visibility:public"],
+ deps = [
+ "//scouting/db",
+ "//scouting/scraping",
+ ],
+)
+
+go_test(
+ name = "rankings_test",
+ srcs = ["rankings_test.go"],
+ data = [
+ "scouting_test_config.json",
+ "//scouting/scraping:test_data",
+ ],
+ embed = [":rankings"],
+ deps = [
+ "//scouting/db",
+ "//scouting/webserver/server",
+ ],
+)
diff --git a/scouting/webserver/rankings/rankings.go b/scouting/webserver/rankings/rankings.go
new file mode 100644
index 0000000..064aa13
--- /dev/null
+++ b/scouting/webserver/rankings/rankings.go
@@ -0,0 +1,92 @@
+package rankings
+
+import (
+ "github.com/frc971/971-Robot-Code/scouting/db"
+ "github.com/frc971/971-Robot-Code/scouting/scraping"
+ "log"
+ "strconv"
+ "strings"
+ "time"
+)
+
+type rankingScraper struct {
+ doneChan chan<- bool
+ checkStopped chan<- bool
+}
+
+type Database interface {
+ AddOrUpdateRankings(db.Ranking) error
+}
+
+func parseTeamKey(teamKey string) (int, error) {
+ // TBA prefixes teams with "frc". Not sure why. Get rid of that.
+ teamKey = strings.TrimPrefix(teamKey, "frc")
+ return strconv.Atoi(teamKey)
+}
+
+func getRankings(database Database, year int32, eventCode string, blueAllianceConfig string) {
+ rankings, err := scraping.AllRankings(year, eventCode, blueAllianceConfig)
+ if err != nil {
+ log.Println("Failed to scrape ranking list: ", err)
+ return
+ }
+
+ for _, rank := range rankings.Rankings {
+ teamKey, err := parseTeamKey(rank.TeamKey)
+
+ if err != nil {
+ log.Println("Failed to parse team key: ", err)
+ continue
+ }
+
+ rankingInfo := db.Ranking{
+ TeamNumber: teamKey,
+ Losses: rank.Records.Losses, Wins: rank.Records.Wins, Ties: rank.Records.Ties,
+ Rank: rank.Rank, Dq: rank.Dq,
+ }
+ err = database.AddOrUpdateRankings(rankingInfo)
+
+ if err != nil {
+ log.Println("Failed to add or update database: ", err)
+ }
+ }
+}
+
+func (scraper *rankingScraper) Start(database Database, year int32, eventCode string, blueAllianceConfig string) {
+ scraper.doneChan = make(chan bool, 1)
+ scraper.checkStopped = make(chan bool, 1)
+
+ go func(database Database, year int32, eventCode string) {
+ // Setting start time to 11 minutes prior so getRankings called instantly when Start() called
+ startTime := time.Now().Add(-11 * time.Minute)
+ for {
+ curTime := time.Now()
+ diff := curTime.Sub(startTime)
+
+ if diff.Minutes() > 10 {
+ getRankings(database, year, eventCode, blueAllianceConfig)
+ startTime = curTime
+ }
+
+ if len(scraper.doneChan) != 0 {
+ break
+ }
+
+ time.Sleep(time.Second)
+ }
+
+ scraper.checkStopped <- true
+ }(database, year, eventCode)
+}
+
+func (scraper *rankingScraper) Stop() {
+ scraper.doneChan <- true
+
+ for {
+ if len(scraper.checkStopped) != 0 {
+ close(scraper.doneChan)
+ close(scraper.checkStopped)
+ break
+ }
+ }
+}
diff --git a/scouting/webserver/rankings/rankings_test.go b/scouting/webserver/rankings/rankings_test.go
new file mode 100644
index 0000000..f47c0a2
--- /dev/null
+++ b/scouting/webserver/rankings/rankings_test.go
@@ -0,0 +1,69 @@
+package rankings
+
+import (
+ "github.com/frc971/971-Robot-Code/scouting/db"
+ "github.com/frc971/971-Robot-Code/scouting/webserver/server"
+ "net/http"
+ "reflect"
+ "testing"
+ "time"
+)
+
+type MockDatabase struct {
+ rankings []db.Ranking
+}
+
+func (database *MockDatabase) AddOrUpdateRankings(data db.Ranking) error {
+ database.rankings = append(database.rankings, data)
+ return nil
+}
+
+func ServeRankings(h http.Handler) http.Handler {
+ fn := func(w http.ResponseWriter, r *http.Request) {
+ r.URL.Path = "scraping/test_data/2016_nytr_rankings.json"
+
+ h.ServeHTTP(w, r)
+ }
+
+ return http.HandlerFunc(fn)
+}
+
+func TestGetRankings(t *testing.T) {
+ database := MockDatabase{}
+ scraper := rankingScraper{}
+ tbaServer := server.NewScoutingServer()
+ tbaServer.Handle("/", ServeRankings(http.FileServer(http.Dir("../../"))))
+ tbaServer.Start(8000)
+ defer tbaServer.Stop()
+
+ scraper.Start(&database, 2016, "nytr", "scouting_test_config.json")
+ defer scraper.Stop()
+
+ for {
+ if len(database.rankings) > 0 {
+ break
+ }
+
+ time.Sleep(time.Second)
+ }
+
+ beginningThreeExpected := []db.Ranking{
+ {TeamNumber: 359, Losses: 1, Wins: 11, Ties: 0, Rank: 1, Dq: 0},
+ {TeamNumber: 5254, Losses: 1, Wins: 11, Ties: 0, Rank: 2, Dq: 0},
+ {TeamNumber: 3990, Losses: 1, Wins: 11, Ties: 0, Rank: 3, Dq: 0},
+ }
+
+ endThreeExpected := []db.Ranking{
+ {TeamNumber: 5943, Losses: 10, Wins: 2, Ties: 0, Rank: 34, Dq: 0},
+ {TeamNumber: 4203, Losses: 10, Wins: 2, Ties: 0, Rank: 35, Dq: 0},
+ {TeamNumber: 5149, Losses: 10, Wins: 2, Ties: 0, Rank: 36, Dq: 0},
+ }
+
+ if !reflect.DeepEqual(beginningThreeExpected, database.rankings[0:3]) {
+ t.Fatal("Got %#v, but expected %#v.", database.rankings[0:3], beginningThreeExpected)
+ }
+
+ if !reflect.DeepEqual(endThreeExpected, database.rankings[33:]) {
+ t.Fatal("Got %#v, but expected %#v.", database.rankings[33:], beginningThreeExpected)
+ }
+}
diff --git a/scouting/webserver/rankings/scouting_test_config.json b/scouting/webserver/rankings/scouting_test_config.json
new file mode 100644
index 0000000..19a1b4f
--- /dev/null
+++ b/scouting/webserver/rankings/scouting_test_config.json
@@ -0,0 +1,4 @@
+{
+ "api_key": "dummy_key_that_is_not_actually_used_in_this_test",
+ "base_url": "http://localhost:8000"
+}
\ No newline at end of file