feat(client): a SubmitQueue client library, with list and watch

MODULE.bazel

@@ -66,6 +66,7 @@ use_repo(
     "org_golang_google_protobuf",
     "org_golang_x_oauth2",
     "org_golang_x_sync",
+    "org_golang_x_term",
     "org_uber_go_fx",
     "org_uber_go_mock",
     "org_uber_go_yarpc",

Makefile

@@ -56,7 +56,10 @@ export SQ_PROVIDER_CONFIG_DIR ?= $(REPO_ROOT)/service/submitqueue/demo/provider/
 DEMO_REPO ?= behinddwalls/sq-demo
 COUNT ?= 3
 FILES ?= 3
+CONCURRENCY ?= 5
 STACKED ?= false
+SINCE ?= 1h
+LIMIT ?= 50
 LAND ?= true
 WATCH ?= true
 QUEUE ?= demo-queue
@@ -153,13 +156,14 @@ clean-proto: ## Clean generated proto files
 	@rm -f $(foreach p,$(PROTO_PACKAGES),$(p)/protopb/*.pb.go $(p)/protopb/*.pb.yarpc.go)
 	@echo "Proto clean complete!"
 
-demo-pr: ## Create N PRs in the demo repo, enqueue each as it is created, and watch (COUNT=3 FILES=3; needs GITHUB_TOKEN)
+demo-pr: ## Create N PRs in the demo repo, enqueue each as it is created, and watch (COUNT=3 FILES=3 CONCURRENCY=5; needs GITHUB_TOKEN)
 	@$(BAZEL) run //service/submitqueue/demo/pr -- \
 		-repo $(DEMO_REPO) \
 		-count $(COUNT) \
 		-files $(FILES) \
+		-concurrency $(CONCURRENCY) \
 		-stacked=$(STACKED) \
-		-gateway $(GATEWAY_ADDR) \
+		-addr $(GATEWAY_ADDR) \
 		-queue $(QUEUE) \
 		-strategy $(STRATEGY) \
 		-land=$(LAND) -watch=$(WATCH)
@@ -227,6 +231,14 @@ land-status: ## Read a landed request's status (SQID=... [QUEUE=demo-queue])
 	@$(BAZEL) run //service/submitqueue/gateway/client:gateway -- \
 		-addr $(GATEWAY_ADDR) status -queue $(QUEUE) -sqid $(SQID)
 
+land-list: ## Show a queue's recent requests as a table (QUEUE=demo-queue SINCE=1h LIMIT=50)
+	@$(BAZEL) run //service/submitqueue/gateway/client:gateway -- \
+		-addr $(GATEWAY_ADDR) list -queue $(QUEUE) -since $(SINCE) -limit $(LIMIT)
+
+land-watch: ## Follow a queue's requests until they settle (QUEUE=demo-queue SINCE=15m LIMIT=50)
+	@$(BAZEL) run //service/submitqueue/gateway/client:gateway -- \
+		-addr $(GATEWAY_ADDR) watch -queue $(QUEUE) -since $(SINCE) -limit $(LIMIT)
+
 license-fix: ## Add missing license headers to source files
 	@$(BAZEL) run //tool/linter/licenseheader -- --fix
 

doc/howto/PROVIDER-E2E.md

@@ -97,12 +97,15 @@ Opening pull requests by hand gets old fast. `demo-pr` creates them, enqueues th
 make demo-pr                      # 3 independent PRs, each enqueued as it is created
 make demo-pr COUNT=8              # more traffic
 make demo-pr FILES=8              # wider changes, more files per PR
+make demo-pr CONCURRENCY=1        # create them one at a time
 make demo-pr STACKED=true         # one stack, enqueued as a single request
 make demo-pr LAND=false           # create only, print the land command
 ```
 
 Each pull request is enqueued the moment it exists, so the queue is already working on the first while the last is still being opened. That overlap is the point: a queue holding one request at a time never batches, never analyzes a conflict against another batch, and never speculates. Nothing is awaited until every request is in.
 
+Independent pull requests are created **five at a time** by default (`CONCURRENCY`). Opening one is several round trips — a branch, a commit per file, the pull request itself — so creating them serially was most of what a large run spent its time on, and it delayed the overlap the demo exists to show. A stack ignores the setting: each of its changes is based on the branch before it, so the next cannot be cut until the previous head exists. Lower it if the provider starts refusing bursts.
+
 The table is there from the start — one row per land request, drawn before the first pull request exists and filled in as the run proceeds. Whatever is happening right now is a single line underneath it, so creating and enqueuing does not scroll the table away:
 
 ```
@@ -129,6 +132,40 @@ The command exits non-zero if any request settles anywhere other than `landed`,
 
 ## Watching it work
 
+The queue itself is readable without creating any traffic:
+
+```bash
+make land-list                       # a table of recent requests
+make land-list SINCE=24h LIMIT=200   # a wider window
+make land-watch                      # follow them until they settle
+```
+
+Both draw the same table `make demo-pr` does — the demo tool and the CLI share it — but against whatever the queue already holds, so watching a queue no longer means adding to it. `land-watch` fixes its set when it starts and exits non-zero if any request in that set finishes anywhere other than `landed`, which makes it usable from a script. A request accepted after the watch begins is not picked up: a watch that grew as the queue did would never finish.
+
+Under the hood these are `client list` and `client watch`, which take a queue and reach any gateway:
+
+```bash
+bazel run //service/submitqueue/gateway/client:gateway -- \
+  -addr sq.example.com:443 -tls list -queue my-queue -since 1h
+```
+
+`-addr` is passed to the dialler untouched, so `dns:///host:port` and `unix:///path.sock` work as well as a plain `host:port`. Transport security is a separate flag rather than part of the address, because gRPC keeps target resolution and credentials apart — there is no `grpcs://` to write.
+
+Bear in mind that a request reads `batched` for the whole of its active life (see above), so a listing of a busy queue is mostly `batched` rows until the pipeline reports its finer stages.
+
+### Authentication
+
+The gateway admits every caller. It is a sandbox stack, and nothing in it checks a credential.
+
+The client can still present one, for a gateway reached through something that does — a proxy, a mesh sidecar, an ingress that terminates auth ahead of the service. It reads `SQ_TOKEN` by default and sends it as `Authorization: Bearer …`; `-token-env` names a different variable, and an unset one sends nothing rather than failing, which is how it stays usable against a stack that wants no credential.
+
+```bash
+SQ_TOKEN=$(cat ~/.sq-token) bazel run //service/submitqueue/gateway/client:gateway -- \
+  -addr sq.example.com:443 -tls list -queue my-queue
+```
+
+### Service logs
+
 ```bash
 docker compose -p submitqueue-provider logs -f runway-service
 ```

go.mod

@@ -14,6 +14,7 @@ require (
 	go.uber.org/zap v1.27.1
 	golang.org/x/oauth2 v0.34.0
 	golang.org/x/sync v0.19.0
+	golang.org/x/term v0.39.0
 	google.golang.org/grpc v1.68.1
 	google.golang.org/protobuf v1.36.10
 	gopkg.in/yaml.v3 v3.0.1

go.sum

@@ -249,6 +249,8 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
 golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY=
+golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

service/submitqueue/demo/pr/BUILD.bazel

@@ -2,17 +2,17 @@ load("@rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
-    srcs = ["main.go"],
+    srcs = [
+        "github.go",
+        "main.go",
+    ],
     importpath = "github.com/uber/submitqueue/service/submitqueue/demo/pr",
     visibility = ["//visibility:private"],
     deps = [
-        "//api/base/change/protopb:go_default_library",
         "//api/base/mergestrategy/protopb:go_default_library",
-        "//api/submitqueue/gateway/protopb:go_default_library",
         "//platform/base/change/github:go_default_library",
-        "//submitqueue/entity:go_default_library",
-        "@org_golang_google_grpc//:go_default_library",
-        "@org_golang_google_grpc//credentials/insecure:go_default_library",
+        "//submitqueue/client:go_default_library",
+        "@org_golang_x_sync//errgroup:go_default_library",
     ],
 )
 
@@ -27,9 +27,7 @@ go_test(
     srcs = ["main_test.go"],
     embed = [":go_default_library"],
     deps = [
-        "//api/submitqueue/gateway/protopb:go_default_library",
         "@com_github_stretchr_testify//assert:go_default_library",
         "@com_github_stretchr_testify//require:go_default_library",
-        "@org_golang_google_grpc//:go_default_library",
     ],
 )

service/submitqueue/demo/pr/github.go

@@ -0,0 +1,125 @@
+// Copyright (c) 2025 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+)
+
+// githubClient is the slice of GitHub's REST API this tool needs: read a
+// branch, create a branch, commit a file, open a pull request.
+type githubClient struct {
+	root  string
+	token string
+	owner string
+	repo  string
+}
+
+func (g *githubClient) branchSHA(ctx context.Context, branch string) (string, error) {
+	var out struct {
+		Object struct {
+			SHA string `json:"sha"`
+		} `json:"object"`
+	}
+	if err := g.do(ctx, http.MethodGet, "/git/ref/heads/"+branch, nil, &out); err != nil {
+		return "", err
+	}
+	return out.Object.SHA, nil
+}
+
+func (g *githubClient) createBranch(ctx context.Context, branch, fromSHA string) error {
+	return g.do(ctx, http.MethodPost, "/git/refs",
+		map[string]string{"ref": "refs/heads/" + branch, "sha": fromSHA}, nil)
+}
+
+// commitFile writes a file on a branch and returns the resulting commit SHA —
+// the commit a change URI pins the pull request to.
+func (g *githubClient) commitFile(ctx context.Context, branch, path, content, message string) (string, error) {
+	body := map[string]string{
+		"message": message,
+		"content": base64.StdEncoding.EncodeToString([]byte(content)),
+		"branch":  branch,
+	}
+	var out struct {
+		Commit struct {
+			SHA string `json:"sha"`
+		} `json:"commit"`
+	}
+	if err := g.do(ctx, http.MethodPut, "/contents/"+path, body, &out); err != nil {
+		return "", err
+	}
+	return out.Commit.SHA, nil
+}
+
+func (g *githubClient) openPR(ctx context.Context, title, head, base string) (int, string, error) {
+	body := map[string]string{"title": title, "head": head, "base": base, "body": "Opened by service/submitqueue/demo/pr."}
+	var out struct {
+		Number  int    `json:"number"`
+		HTMLURL string `json:"html_url"`
+	}
+	if err := g.do(ctx, http.MethodPost, "/pulls", body, &out); err != nil {
+		return 0, "", err
+	}
+	return out.Number, out.HTMLURL, nil
+}
+
+// do issues one authenticated request against the repository, decoding into out
+// when it is non-nil.
+func (g *githubClient) do(ctx context.Context, method, path string, body any, out any) error {
+	endpoint := fmt.Sprintf("%s/repos/%s/%s%s", g.root, g.owner, g.repo, path)
+
+	var payload []byte
+	if body != nil {
+		var err error
+		if payload, err = json.Marshal(body); err != nil {
+			return fmt.Errorf("encode request for %s: %w", endpoint, err)
+		}
+	}
+
+	req, err := http.NewRequestWithContext(ctx, method, endpoint, bytes.NewReader(payload))
+	if err != nil {
+		return fmt.Errorf("build request for %s: %w", endpoint, err)
+	}
+	req.Header.Set("Accept", "application/vnd.github+json")
+	req.Header.Set("Authorization", "Bearer "+g.token)
+	if payload != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("%s %s: %w", method, endpoint, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		var detail bytes.Buffer
+		_, _ = detail.ReadFrom(resp.Body)
+		return fmt.Errorf("%s %s returned %s: %s", method, endpoint, resp.Status, strings.TrimSpace(detail.String()))
+	}
+	if out == nil {
+		return nil
+	}
+	if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
+		return fmt.Errorf("decode response from %s: %w", endpoint, err)
+	}
+	return nil
+}