scouting: Add an endpoint for populating the match schedule
This patch combines the scraping library with the scouting webserver.
There's now also a new end point for the web page (or debug CLI tool)
to ask the server to fetch the match list. The end point is
`/requests/refresh_match_list`.
All the tests are updated. The `cli_test` downloads a 2016 ny_tr match
list that I downloaded from TBA. It should be a decent integration
test as it uses representative data.
Signed-off-by: Philipp Schrader <philipp.schrader@gmail.com>
Change-Id: I6c540590521b00887eb2ddde2a9369875c659551
diff --git a/scouting/scraping/scrape.go b/scouting/scraping/scrape.go
index fa20f7b..170fe50 100644
--- a/scouting/scraping/scrape.go
+++ b/scouting/scraping/scrape.go
@@ -4,15 +4,17 @@
import (
"encoding/json"
"errors"
+ "fmt"
"io/ioutil"
- "log"
"net/http"
"os"
+ "strconv"
)
// Stores the TBA API key to access the API.
-type params struct {
- ApiKey string `json:"api_key"`
+type scrapingConfig struct {
+ ApiKey string `json:"api_key"`
+ BaseUrl string `json:"base_url"`
}
// Takes in year and FIRST event code and returns all matches in that event according to TBA.
@@ -22,64 +24,64 @@
//{
// api_key:"myTBAapiKey"
//}
-func AllMatches(year, eventCode, filePath string) ([]Match, error) {
- if filePath == "" {
- filePath = os.Getenv("BUILD_WORKSPACE_DIRECTORY") + "/scouting_config.json"
+func AllMatches(year int32, eventCode, configPath string) ([]Match, error) {
+ if configPath == "" {
+ configPath = os.Getenv("BUILD_WORKSPACE_DIRECTORY") + "/scouting_config.json"
}
+
// Takes the filepath and grabs the api key from the json.
- content, err := ioutil.ReadFile(filePath)
+ content, err := ioutil.ReadFile(configPath)
if err != nil {
- log.Fatal(err)
+ return nil, errors.New(fmt.Sprint("Failed to open config at ", configPath, ": ", err))
}
// Parses the JSON parameters into a struct.
- var passed_params params
- error := json.Unmarshal([]byte(content), &passed_params)
- if error != nil {
- log.Fatalf("You forgot to add the api_key parameter in the json file")
- log.Fatalf("%s", err)
+ var config scrapingConfig
+ if err := json.Unmarshal([]byte(content), &config); err != nil {
+ return nil, errors.New(fmt.Sprint("Failed to parse config file as JSON: ", err))
+ }
+
+ // Perform some basic validation on the data.
+ if config.ApiKey == "" {
+ return nil, errors.New("Missing 'api_key' in config JSON.")
+ }
+ if config.BaseUrl == "" {
+ config.BaseUrl = "https://www.thebluealliance.com"
}
// Create the TBA event key for the year and event code.
- eventKey := year + eventCode
-
- // Create the client for HTTP requests.
- client := &http.Client{}
+ eventKey := strconv.Itoa(int(year)) + eventCode
// Create a get request for the match info.
- req, err := http.NewRequest("GET", "https://www.thebluealliance.com/api/v3/event/"+eventKey+"/matches", nil)
-
+ req, err := http.NewRequest("GET", config.BaseUrl+"/api/v3/event/"+eventKey+"/matches", nil)
if err != nil {
- return nil, errors.New("failed to build http request")
+ return nil, errors.New(fmt.Sprint("Failed to build http request: ", err))
}
// Add the auth key header to the request.
- req.Header.Add("X-TBA-Auth-Key", passed_params.ApiKey)
+ req.Header.Add("X-TBA-Auth-Key", config.ApiKey)
- // Make the API request
+ // Make the API request.
+ client := &http.Client{}
resp, err := client.Do(req)
-
if err != nil {
- return nil, err
+ return nil, errors.New(fmt.Sprint("Failed to make TBA API request: ", err))
}
- if resp.Status != "200 OK" {
- return nil, errors.New("Recieved a status of " + resp.Status + " expected : 200 OK")
- }
-
- // Wait until the response is done.
defer resp.Body.Close()
+ if resp.StatusCode != 200 {
+ return nil, errors.New(fmt.Sprint("Got unexpected status code from TBA API request: ", resp.Status))
+ }
// Get all bytes from response body.
bodyBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
- return nil, errors.New("failed to read response body with error :" + err.Error())
+ return nil, errors.New(fmt.Sprint("Failed to read TBA API response: ", err))
}
var matches []Match
// Unmarshal json into go usable format.
- jsonError := json.Unmarshal([]byte(bodyBytes), &matches)
- if jsonError != nil {
- return nil, errors.New("failed to unmarshal json recieved from TBA")
+ if err := json.Unmarshal([]byte(bodyBytes), &matches); err != nil {
+ return nil, errors.New(fmt.Sprint("Failed to parse JSON received from TBA: ", err))
}
return matches, nil