Add a golang program used to trigger buildkite from gerrit
You need an API key and an account to make changes, but other than
that, it should just work.
It also implements a webhook to detect pass/fail events. Turns out
buildkite doesn't really like doing anything other than webhooks for
this for what we are doing.
I'm sure I'm not the cleanest on my golang, but it works.
Change-Id: I78707301c26006c73ceee52d2e996bb49e9e7899
diff --git a/tools/ci/buildkite_gerrit_trigger.go b/tools/ci/buildkite_gerrit_trigger.go
new file mode 100644
index 0000000..eb31a92
--- /dev/null
+++ b/tools/ci/buildkite_gerrit_trigger.go
@@ -0,0 +1,419 @@
+package main
+
+import (
+ "bufio"
+ "bytes"
+ "encoding/json"
+ "flag"
+ "fmt"
+ "github.com/buildkite/go-buildkite/buildkite"
+ "io/ioutil"
+ "log"
+ "net/http"
+ "os/exec"
+ "regexp"
+ "strings"
+ "sync"
+)
+
+type Commit struct {
+ Sha1 string
+ ChangeId string
+ ChangeNumber int
+ Patchset int
+}
+
+type State struct {
+ // This mutex needs to be locked across anything which generates a uuid or accesses Commits.
+ mu sync.Mutex
+ // Mapping from build UUID from buildkite to the Commit information which triggered it.
+ Commits map[string]Commit
+ User string
+ Key string
+ // Webhook token expected to service requests
+ Token string
+}
+
+type Approval struct {
+ Type string `json:"type"`
+ Description string `json:"description"`
+ Value string `json:"value"`
+ OldValue string `json:"oldValue"`
+}
+
+type User struct {
+ Name string `json:"name"`
+ Email string `json:"email,omitempty"`
+ Username string `json:"username,omitempty"`
+}
+
+type PatchSet struct {
+ Number int `json:"number"`
+ Revision string `json:"revision"`
+ Parents []string `json:"parents"`
+ Ref string `json:"ref"`
+ Uploader User `json:"uploader"`
+ CreatedOn int `json:"createdOn"`
+ Author User `json:"author"`
+ Kind string `json:"kind,omitempty"`
+ SizeInsertions int `json:"sizeInsertions,omitempty"`
+ SizeDeletions int `json:"sizeDeletions,omitempty"`
+}
+
+type Change struct {
+ Project string `json:"project"`
+ Branch string `json:"branch"`
+ ID string `json:"id"`
+ Number int `json:"number"`
+ Subject string `json:"subject"`
+ Owner User `json:"owner"`
+ URL string `json:"url"`
+ CommitMessage string `json:"commitMessage"`
+ CreatedOn int `json:"createdOn"`
+ Status string `json:"status"`
+}
+
+type ChangeKey struct {
+ ID string `json:"id"`
+}
+
+type RefUpdate struct {
+ OldRev string `json:"oldRev"`
+ NewRev string `json:"newRev"`
+ RefName string `json:"refName"`
+ Project string `json:"project"`
+}
+
+type EventInfo struct {
+ Author *User `json:"author"`
+ Uploader *User `json:"uploader"`
+ Submitter User `json:"submitter,omitempty"`
+ Approvals []Approval `json:"approvals,omitempty"`
+ Comment string `json:"comment,omitempty"`
+ PatchSet *PatchSet `json:"patchSet"`
+ Change *Change `json:"change"`
+ Project string `json:"project"`
+ RefName string `json:"refName"`
+ ChangeKey ChangeKey `json:"changeKey"`
+ RefUpdate *RefUpdate `json:"refUpdate"`
+ Type string `json:"type"`
+ EventCreatedOn int `json:"eventCreatedOn"`
+}
+
+// Simple application to poll Gerrit for events and trigger builds on buildkite when one happens.
+
+// Handles a gerrit event and triggers buildkite accordingly.
+func (s *State) handleEvent(eventInfo EventInfo, client *buildkite.Client) {
+ // Only work on 971-Robot-Code
+ if eventInfo.Project != "971-Robot-Code" {
+ log.Printf("Ignoring project: '%s'\n", eventInfo.Project)
+ return
+ }
+
+ // Find the change id, change number, patchset revision
+ if eventInfo.Change == nil {
+ log.Println("Failed to find Change")
+ return
+ }
+
+ if eventInfo.PatchSet == nil {
+ log.Println("Failed to find Change")
+ return
+ }
+
+ log.Printf("Got a matching change of %s %s %d,%d\n",
+ eventInfo.Change.ID, eventInfo.PatchSet.Revision, eventInfo.Change.Number, eventInfo.PatchSet.Number)
+
+ // Triggering a build creates a UUID, and we can see events back from the webhook before the command returns. Lock across the command so nothing access Commits while the new UUID is being added.
+ s.mu.Lock()
+
+ var user *User
+ if eventInfo.Author != nil {
+ user = eventInfo.Author
+ } else if eventInfo.Uploader != nil {
+ user = eventInfo.Uploader
+ } else {
+ log.Fatalf("Failed to find Author or Uploader")
+ }
+
+ // Trigger the build.
+ if build, _, err := client.Builds.Create(
+ "spartan-robotics", "971-robot-code", &buildkite.CreateBuild{
+ Commit: eventInfo.PatchSet.Revision,
+ Branch: eventInfo.Change.ID,
+ Author: buildkite.Author{
+ Name: user.Name,
+ Email: user.Email,
+ },
+ Env: map[string]string{
+ "GERRIT_CHANGE_NUMBER": fmt.Sprintf("%d", eventInfo.Change.Number),
+ "GERRIT_PATCH_NUMBER": fmt.Sprintf("%d", eventInfo.PatchSet.Number),
+ },
+ }); err == nil {
+
+ if build.ID != nil {
+ log.Printf("Scheduled build %s\n", *build.ID)
+ s.Commits[*build.ID] = Commit{
+ Sha1: eventInfo.PatchSet.Revision,
+ ChangeId: eventInfo.Change.ID,
+ ChangeNumber: eventInfo.Change.Number,
+ Patchset: eventInfo.PatchSet.Number,
+ }
+ }
+ s.mu.Unlock()
+
+ if data, err := json.MarshalIndent(build, "", "\t"); err != nil {
+ log.Fatalf("json encode failed: %s", err)
+ } else {
+ log.Printf("%s\n", string(data))
+ }
+
+ // Now remove the verified from Gerrit and post the link.
+ cmd := exec.Command("ssh",
+ "-p",
+ "29418",
+ "-i",
+ s.Key,
+ s.User+"@software.frc971.org",
+ "gerrit",
+ "review",
+ "-m",
+ fmt.Sprintf("\"Build Started: %s\"", *build.WebURL),
+ "--verified",
+ "0",
+ fmt.Sprintf("%d,%d", eventInfo.Change.Number, eventInfo.PatchSet.Number))
+
+ log.Printf("Running 'ssh -p 29418 -i %s %s@software.frc971.org gerrit review -m '\"Build Started: %s\"' --verified 0 %d,%d' and waiting for it to finish...",
+ s.Key, s.User,
+ *build.WebURL, eventInfo.Change.Number, eventInfo.PatchSet.Number)
+ if err := cmd.Run(); err != nil {
+ log.Printf("Command failed with error: %v", err)
+ }
+ } else {
+ s.mu.Unlock()
+ log.Fatalf("Failed to trigger build: %s", err)
+ }
+
+}
+
+type Build struct {
+ ID string `json:"id,omitempty"`
+ GraphqlId string `json:"graphql_id,omitempty"`
+ URL string `json:"url,omitempty"`
+ WebURL string `json:"web_url,omitempty"`
+ Number int `json:"number,omitempty"`
+ State string `json:"state,omitempty"`
+ Blocked bool `json:"blocked,omitempty"`
+ BlockedState string `json:"blocked_state,omitempty"`
+ Message string `json:"message,omitempty"`
+ Commit string `json:"commit"`
+ Branch string `json:"branch"`
+ Source string `json:"source,omitempty"`
+ CreatedAt string `json:"created_at,omitempty"`
+ ScheduledAt string `json:"scheduled_at,omitempty"`
+ StartedAt string `json:"started_at,omitempty"`
+ FinishedAt string `json:"finished_at,omitempty"`
+}
+
+type BuildkiteWebhook struct {
+ Event string `json:"event"`
+ Build Build `json:"build"`
+}
+
+func (s *State) handle(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/" {
+ http.Error(w, "404 not found.", http.StatusNotFound)
+ return
+ }
+
+ switch r.Method {
+ case "POST":
+ if r.Header.Get("X-Buildkite-Token") != s.Token {
+ http.Error(w, "Invalid token", http.StatusBadRequest)
+ return
+ }
+
+ var data []byte
+ var err error
+ if data, err = ioutil.ReadAll(r.Body); err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+
+ log.Println(string(data))
+
+ var webhook BuildkiteWebhook
+
+ if err := json.Unmarshal(data, &webhook); err != nil {
+ log.Fatalf("json decode failed: %s", err)
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+
+ // We've successfully received the webhook. Spawn a goroutine in case the mutex is blocked so we don't block this thread.
+ f := func() {
+ if webhook.Event == "build.finished" {
+ var commit *Commit
+ {
+ s.mu.Lock()
+ if c, ok := s.Commits[webhook.Build.ID]; ok {
+ commit = &c
+ delete(s.Commits, webhook.Build.ID)
+ }
+ s.mu.Unlock()
+ }
+
+ if commit == nil {
+ log.Printf("Unknown commit, ID: %s", webhook.Build.ID)
+ } else {
+ var verify string
+ var status string
+
+ if webhook.Build.State == "passed" {
+ verify = "+1"
+ status = "Succeeded"
+ } else {
+ verify = "-1"
+ status = "Failed"
+ }
+
+ cmd := exec.Command("ssh",
+ "-p",
+ "29418",
+ "-i",
+ s.Key,
+ s.User+"@software.frc971.org",
+ "gerrit",
+ "review",
+ "-m",
+ fmt.Sprintf("\"Build %s: %s\"", status, webhook.Build.WebURL),
+ "--verified",
+ verify,
+ fmt.Sprintf("%d,%d", commit.ChangeNumber, commit.Patchset))
+
+ log.Printf("Running 'ssh -p 29418 -i %s %s@software.frc971.org gerrit review -m '\"Build %s: %s\"' --verified %s %d,%d' and waiting for it to finish...",
+ s.Key, s.User,
+ status, webhook.Build.WebURL, verify, commit.ChangeNumber, commit.Patchset)
+ if err := cmd.Run(); err != nil {
+ log.Printf("Command failed with error: %v", err)
+ }
+
+ }
+ if webhook.Build.State == "passed" {
+ log.Printf("Passed build %s: %s", webhook.Build.ID, webhook.Build.Commit)
+ } else {
+ log.Printf("Failed build %s: %s", webhook.Build.ID, webhook.Build.Commit)
+ }
+ }
+ }
+
+ go f()
+
+ log.Printf("%s: %s %s %s\n", webhook.Event, webhook.Build.ID, webhook.Build.Commit, webhook.Build.Branch)
+
+ fmt.Fprintf(w, "")
+
+ default:
+ internalError := http.StatusInternalServerError
+ http.Error(w, "Invalid method", internalError)
+ log.Printf("Invalid method %s", r.Method)
+ }
+}
+
+func main() {
+ apiToken := flag.String("token", "", "API token")
+ webhookToken := flag.String("webhook_token", "", "Expected webhook token")
+ user := flag.String("user", "buildkite", "User to be in gerrit")
+ key := flag.String("key", "~/.ssh/buildkite", "SSH key to use to connect to gerrit")
+ debug := flag.Bool("debug", false, "Enable debugging")
+
+ flag.Parse()
+
+ state := State{
+ Key: *key,
+ User: *user,
+ Commits: make(map[string]Commit),
+ Token: *webhookToken,
+ }
+
+ f := func() {
+ http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+ state.handle(w, r)
+ })
+ log.Println("Starting webhook server on 10005\n")
+ if err := http.ListenAndServe(":10005", nil); err != nil {
+ log.Fatal(err)
+ }
+ }
+
+ if *apiToken == "nope" {
+ log.Println("Only starting server")
+ f()
+ } else {
+ go f()
+ }
+
+ config, err := buildkite.NewTokenConfig(*apiToken, *debug)
+
+ if err != nil {
+ log.Fatalf("client config failed: %s", err)
+ }
+
+ client := buildkite.NewClient(config.Client())
+
+ for {
+ args := fmt.Sprintf("-o ServerAliveInterval=10 -o ServerAliveCountMax=3 -i %s -p 29418 %s@software.frc971.org gerrit stream-events", state.Key, state.User)
+ cmd := exec.Command("ssh", strings.Split(args, " ")...)
+
+ stdout, _ := cmd.StdoutPipe()
+ cmd.Start()
+
+ scanner := bufio.NewScanner(stdout)
+ scanner.Split(bufio.ScanLines)
+ for scanner.Scan() {
+ m := scanner.Text()
+
+ log.Println(m)
+
+ var eventInfo EventInfo
+ dec := json.NewDecoder(bytes.NewReader([]byte(m)))
+ dec.DisallowUnknownFields()
+
+ if err := dec.Decode(&eventInfo); err != nil {
+ log.Printf("Failed to parse JSON: %e\n", err)
+ continue
+ }
+
+ log.Printf("Got an event of type: '%s'\n", eventInfo.Type)
+ switch eventInfo.Type {
+ case "assignee-changed":
+ case "change-abandoned":
+ case "change-deleted":
+ case "change-merged":
+ // TODO(austin): Trigger a master build?
+ case "change-restored":
+ case "comment-added":
+ if matched, _ := regexp.MatchString(`(?m)^retest$`, eventInfo.Comment); !matched {
+ continue
+ }
+
+ state.handleEvent(eventInfo, client)
+ case "dropped-output":
+ case "hashtags-changed":
+ case "project-created":
+ case "patchset-created":
+ state.handleEvent(eventInfo, client)
+ case "ref-updated":
+ case "reviewer-added":
+ case "reviewer-deleted":
+ case "topic-changed":
+ case "wip-state-changed":
+ case "private-state-changed":
+ case "vote-deleted":
+ default:
+ log.Println("Unknown case")
+ }
+ }
+ cmd.Wait()
+ }
+}