Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 21 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ define assert_clean
fi
endef

.PHONY: build build-all-linux build-runway-linux build-submitqueue-gateway-linux build-submitqueue-orchestrator-linux build-stovepipe-linux build-stovepipe-linux-debug check-gazelle check-mocks check-tidy clean clean-proto deps e2e-test fmt gazelle integration-test integration-test-submitqueue-consumer integration-test-extensions integration-test-submitqueue-gateway integration-test-submitqueue-orchestrator license-fix lint lint-fmt lint-license local-init-runway-queue-schema local-init-stovepipe-schemas local-runway-start local-runway-stop local-submitqueue-clean local-submitqueue-gateway-start local-submitqueue-gateway-stop local-init-submitqueue-schemas local-submitqueue-logs local-submitqueue-orchestrator-start local-submitqueue-orchestrator-stop local-submitqueue-ps local-submitqueue-restart local-submitqueue-start local-stop local-stovepipe-debug-start local-stovepipe-logs local-stovepipe-start local-stovepipe-stop mocks proto query-deps query-targets run-client-runway run-client-submitqueue-gateway run-client-submitqueue-orchestrator run-client-stovepipe run-queue-admin test test-no-cache tidy tidy-bazel tidy-go help
.PHONY: build build-all-linux build-runway-linux build-submitqueue-gateway-client build-submitqueue-gateway-linux build-submitqueue-gateway-server build-submitqueue-orchestrator-linux build-stovepipe-linux build-stovepipe-linux-debug check-gazelle check-mocks check-tidy clean clean-proto deps e2e-test fmt gazelle integration-test integration-test-submitqueue-consumer integration-test-extensions integration-test-submitqueue-gateway integration-test-submitqueue-orchestrator license-fix lint lint-binary lint-fmt lint-license local-init-runway-queue-schema local-init-stovepipe-schemas local-runway-start local-runway-stop local-submitqueue-clean local-submitqueue-gateway-start local-submitqueue-gateway-stop local-init-submitqueue-schemas local-submitqueue-logs local-submitqueue-orchestrator-start local-submitqueue-orchestrator-stop local-submitqueue-ps local-submitqueue-restart local-submitqueue-start local-stop local-stovepipe-debug-start local-stovepipe-logs local-stovepipe-start local-stovepipe-stop mocks proto query-deps query-targets run-client-runway run-client-submitqueue-gateway run-client-submitqueue-orchestrator run-client-stovepipe run-queue-admin test test-no-cache tidy tidy-bazel tidy-go help


build: ## Build all services and examples
Expand All @@ -99,6 +99,14 @@ build-runway-linux: ## Build Runway Linux binary for Docker
cp -f bazel-bin/service/runway/server/runway .docker-bin/runway
@echo "Runway Linux binary ready at .docker-bin/runway"

build-submitqueue-gateway-client: ## Build the gateway client CLI for the host platform into bin/client
@echo "Building gateway client..."
@$(BAZEL) build //service/submitqueue/gateway/client:gateway
@mkdir -p bin
@cp -f bazel-bin/service/submitqueue/gateway/client/gateway_/gateway bin/client 2>/dev/null || \
cp -f bazel-bin/service/submitqueue/gateway/client/gateway bin/client
@echo "Gateway client ready at bin/client"

build-submitqueue-gateway-linux: ## Build Gateway Linux binary for Docker
@echo "Building Gateway Linux binary for Docker..."
@$(BAZEL) build --platforms=@rules_go//go/toolchain:linux_amd64 //service/submitqueue/gateway/server:gateway
Expand All @@ -107,6 +115,14 @@ build-submitqueue-gateway-linux: ## Build Gateway Linux binary for Docker
cp -f bazel-bin/service/submitqueue/gateway/server/gateway .docker-bin/gateway
@echo "Gateway Linux binary ready at .docker-bin/gateway"

build-submitqueue-gateway-server: ## Build the gateway server for the host platform into bin/server
@echo "Building gateway server..."
@$(BAZEL) build //service/submitqueue/gateway/server:gateway
@mkdir -p bin
@cp -f bazel-bin/service/submitqueue/gateway/server/gateway_/gateway bin/server 2>/dev/null || \
cp -f bazel-bin/service/submitqueue/gateway/server/gateway bin/server
@echo "Gateway server ready at bin/server"

build-submitqueue-orchestrator-linux: ## Build Orchestrator Linux binary for Docker
@echo "Building Orchestrator Linux binary for Docker..."
@$(BAZEL) build --platforms=@rules_go//go/toolchain:linux_amd64 //service/submitqueue/orchestrator/server:orchestrator
Expand Down Expand Up @@ -242,9 +258,12 @@ land-watch: ## Follow a queue's requests until they settle (QUEUE=demo-queue SIN
license-fix: ## Add missing license headers to source files
@$(BAZEL) run //tool/linter/licenseheader -- --fix

lint: lint-fmt lint-license lint-message-id lint-queue-shard ## Run all linters
lint: lint-binary lint-fmt lint-license lint-message-id lint-queue-shard ## Run all linters
@echo "All lint checks passed."

lint-binary: ## Check no binary file is tracked in the repository
@$(BAZEL) run //tool/linter/binaryfile

lint-fmt: fmt ## Check code formatting (fails if unformatted)
$(call assert_clean,make fmt)
@echo "All code is properly formatted."
Expand Down
24 changes: 24 additions & 0 deletions tool/linter/binaryfile/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
load("@rules_go//go:def.bzl", "go_binary", "go_library", "go_test")

go_library(
name = "go_default_library",
srcs = ["main.go"],
importpath = "github.com/uber/submitqueue/tool/linter/binaryfile",
visibility = ["//visibility:private"],
)

go_binary(
name = "binaryfile",
embed = [":go_default_library"],
visibility = ["//visibility:public"],
)

go_test(
name = "go_default_test",
srcs = ["main_test.go"],
embed = [":go_default_library"],
deps = [
"@com_github_stretchr_testify//assert:go_default_library",
"@com_github_stretchr_testify//require:go_default_library",
],
)
182 changes: 182 additions & 0 deletions tool/linter/binaryfile/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
// Copyright (c) 2026 Uber Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Command binaryfile checks that no binary file is tracked in the repository.
//
// The build is Bazel-driven and every artifact it produces lands in an ignored
// directory — bazel-bin/, bin/, or .docker-bin/ — so a tracked binary is always
// a mistake. The usual cause is an ad-hoc `go build ./service/...` run from the
// repo root: it names its executable after the package directory and writes it
// to the working directory, where a broad `git add` sweeps it up.
//
// Checking rather than ignoring is deliberate. A .gitignore entry would stop
// the file being committed but would also stop git mentioning it at all, so the
// mistake becomes invisible and the stray artifact simply accumulates. A check
// that fails names the file and says what to do instead.
//
// Detection follows git's own heuristic: a file is binary if a NUL byte appears
// in its leading bytes.
package main

import (
"bytes"
"flag"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
)

// sniffLen is how many leading bytes are examined for a NUL. It matches the
// window git uses for the same decision, which is large enough to cover any
// text header an executable format might begin with.
const sniffLen = 8000

// allowed lists tracked paths that are legitimately binary, relative to the
// repository root. It is empty because nothing in the repository is: the tree
// is source, schemas, and generated Go. An entry belongs here only when a
// binary genuinely has to be versioned — a test fixture that cannot be built,
// or an image a document renders — never to silence a stray build artifact.
var allowed = map[string]bool{}

// violation is one tracked file that is binary.
type violation struct {
path string
size int64
}

func main() {
flag.Parse()

root, err := findRepoRoot()
if err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1)
}

files, err := trackedFiles(root)
if err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1)
}

var violations []violation
var checked int
for _, path := range files {
if allowed[path] {
continue
}
info, err := os.Lstat(filepath.Join(root, path))
// A tracked path that is missing or is a symlink has no contents of its
// own to judge; skip it rather than failing the whole run.
if err != nil || !info.Mode().IsRegular() {
continue
}
checked++

binary, err := isBinaryFile(filepath.Join(root, path))
if err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1)
}
if binary {
violations = append(violations, violation{path: path, size: info.Size()})
}
}

if len(violations) > 0 {
fmt.Fprintf(os.Stderr, "%d binary file(s) are tracked:\n\n", len(violations))
for _, v := range violations {
fmt.Fprintf(os.Stderr, " %s (%d bytes)\n", v.path, v.size)
}
fmt.Fprintf(os.Stderr, "\nThe build is Bazel-driven and writes to bazel-bin/, bin/, and\n")
fmt.Fprintf(os.Stderr, ".docker-bin/, all of which are ignored, so a tracked binary is a\n")
fmt.Fprintf(os.Stderr, "mistake. If this is a stray `go build` output, delete it and use\n")
fmt.Fprintf(os.Stderr, "Bazel instead: `make build`, or `make run-client-submitqueue-gateway`\n")
fmt.Fprintf(os.Stderr, "to run the client without producing a binary at all.\n")
os.Exit(1)
}

fmt.Printf("All %d tracked files are text.\n", checked)
}

// isBinaryFile reports whether the file at path is binary, reading no more than
// the leading sniffLen bytes.
func isBinaryFile(path string) (bool, error) {
file, err := os.Open(path)
if err != nil {
return false, fmt.Errorf("failed to open %s: %w", path, err)
}
defer func() { _ = file.Close() }()

buf := make([]byte, sniffLen)
n, err := file.Read(buf)
if err != nil && n == 0 {
// A read that returns nothing, including io.EOF on an empty file, leaves
// an empty window, which isBinary correctly reports as text.
return isBinary(nil), nil
}
return isBinary(buf[:n]), nil
}

// isBinary reports whether a leading window of a file's contents looks binary,
// which is true exactly when it contains a NUL byte. An empty window is text:
// an empty file has nothing to make it binary.
func isBinary(window []byte) bool {
return bytes.IndexByte(window, 0) >= 0
}

// trackedFiles returns every path git tracks, relative to root.
//
// The check is about what is committed rather than what is present, so the file
// list comes from the index; a stray artifact that is untracked is a local
// matter and not this linter's business.
func trackedFiles(root string) ([]string, error) {
cmd := exec.Command("git", "-C", root, "ls-files", "-z")
out, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf("failed to list tracked files: %w", err)
}

var files []string
for _, path := range strings.Split(string(out), "\x00") {
if path != "" {
files = append(files, path)
}
}
return files, nil
}

func findRepoRoot() (string, error) {
// Bazel `run` executes from the runfiles tree; BUILD_WORKSPACE_DIRECTORY
// points back at the source tree.
if dir := os.Getenv("BUILD_WORKSPACE_DIRECTORY"); dir != "" {
return dir, nil
}
dir, err := os.Getwd()
if err != nil {
return "", err
}
for {
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
return dir, nil
}
parent := filepath.Dir(dir)
if parent == dir {
return "", fmt.Errorf("could not find repository root (no go.mod found)")
}
dir = parent
}
}
94 changes: 94 additions & 0 deletions tool/linter/binaryfile/main_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
// Copyright (c) 2026 Uber Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
"os"
"path/filepath"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestIsBinary(t *testing.T) {
tests := []struct {
name string
window []byte
want bool
}{
{name: "empty is text", window: nil, want: false},
{name: "ascii source is text", window: []byte("package main\n"), want: false},
{name: "utf-8 is text", window: []byte("// © Uber — naïve\n"), want: false},
{name: "crlf is text", window: []byte("a\r\nb\r\n"), want: false},
{name: "high bytes without NUL are text", window: []byte{0x80, 0xfe, 0xff}, want: false},
{name: "leading NUL is binary", window: []byte{0x00, 'a'}, want: true},
{name: "trailing NUL is binary", window: []byte{'a', 0x00}, want: true},
{name: "elf header is binary", window: []byte{0x7f, 'E', 'L', 'F', 0x02, 0x00}, want: true},
{name: "mach-o header is binary", window: []byte{0xcf, 0xfa, 0xed, 0xfe, 0x0c, 0x00}, want: true},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, isBinary(tt.window))
})
}
}

func TestIsBinaryFile(t *testing.T) {
dir := t.TempDir()

tests := []struct {
name string
contents []byte
want bool
}{
{name: "empty file is text", contents: []byte{}, want: false},
{name: "go source is text", contents: []byte("package main\n\nfunc main() {}\n"), want: false},
{name: "executable is binary", contents: append([]byte{0x7f, 'E', 'L', 'F'}, make([]byte, 512)...), want: true},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
path := filepath.Join(dir, tt.name)
require.NoError(t, os.WriteFile(path, tt.contents, 0o600))

got, err := isBinaryFile(path)
require.NoError(t, err)
assert.Equal(t, tt.want, got)
})
}
}

func TestIsBinaryFileOnlyReadsTheLeadingWindow(t *testing.T) {
// A NUL past the sniff window is not reached, which is what bounds the
// linter's cost on a large text file rather than a claim about the file.
path := filepath.Join(t.TempDir(), "late-nul")
contents := append(make([]byte, 0, sniffLen+2), []byte("package main\n")...)
for len(contents) < sniffLen {
contents = append(contents, 'x')
}
contents = append(contents, 0x00)
require.NoError(t, os.WriteFile(path, contents, 0o600))

got, err := isBinaryFile(path)
require.NoError(t, err)
assert.False(t, got)
}

func TestIsBinaryFileMissing(t *testing.T) {
_, err := isBinaryFile(filepath.Join(t.TempDir(), "absent"))
require.Error(t, err)
}
Loading