bravebot/Makefile at main · brave/bravebot

462 lines (426 loc) · 20.8 KB
BINARY = bravebot
VERSION = $(shell sed -nE 's/^version[[:space:]]*=[[:space:]]*"([0-9]+\.[0-9]+\.[0-9]+)".*/\1/p' Cargo.toml | head -n 1)
TAG = v$(VERSION)
# Which remote a release is tagged against. `origin` is right for a clone of this repository
# and wrong for a clone of a fork of it, where it names the fork: a tag pushed there is a tag
# the releases page never sees, and it cannot be moved afterwards because the tag ruleset
# refuses an update. Set it per clone rather than per release, with
# `git config bravebot.releaseRemote upstream`.
RELEASE_REMOTE = $(or $(shell git config bravebot.releaseRemote),origin)
# The minimum toolchain CI builds against, declared once in Cargo.toml.
MSRV = $(shell sed -nE 's/^rust-version[[:space:]]*=[[:space:]]*"([0-9.]+)".*/\1/p' Cargo.toml | head -n 1)
# Every file that states the version, which is what a bump rewrites and commits.
VERSION_FILES = Cargo.toml Cargo.lock package.json package-lock.json
# Forwarded into the cross-build container, which does not inherit the host environment.
BUILD_ENV = SERVICES_KEY_AICHAT BRAVE_SERVICES_KEY_ID BRAVE_AI_CHAT_ENDPOINT \
            BRAVE_AI_CHAT_PREMIUM_ENDPOINT BRAVE_AI_CHAT_DEFAULT_MODEL \
            BRAVEBOT_ALLOW_UNCONFIGURED_BUILD
.PHONY: help
	@echo "bravebot $(VERSION)"
	@echo "Development:"
	@echo "  make init                  Configure bravebot, Claude Code, Codex and Cursor; install hooks"
	@echo "  make hooks                 Point git at the checked-in git hooks"
	@echo "  make build                 Debug build"
	@echo "  make test                  Run all tests"
	@echo "  make check                 Format check, clippy, tests, and toolchain age"
	@echo "  make check-spec            Check docs/specs against the implementation"
	@echo "  make write-unverified      Write unverified-clauses.txt, which check-spec holds it to"
	@echo "  make check-reviewdog       The PR security scan, on this branch's changes"
	@echo "  make check-reviewdog-full  The same scan, over the whole tree"
	@echo "  make check-npm             Install from the lockfile and lint it, as CI does"
	@echo "  make check-deps            Advisories, licences, duplicate versions, and sources"
	@echo "  make check-msrv            Build against the declared minimum toolchain ($(MSRV))"
	@echo "  make check-windows         Lint the Windows target that ships, cross-compiled"
	@echo "  make check-all             Every check any CI enforces, including the security scan"
	@echo "  make locales               What each translation has, and what it is missing"
	@echo "  make check-linux           The same checks on Linux, current stable toolchain"
	@echo "  make fmt                   Apply formatting"
	@echo "  make aws-logout            End every cached AWS SSO session, to test signing in again"
	@echo "Reproducible cross-builds (requires Docker):"
	@echo "  make all-platforms  Every target below"
	@echo "  make darwin-arm64   macOS Apple silicon"
	@echo "  make darwin-amd64   macOS Intel"
	@echo "  make linux-amd64    Linux x86_64"
	@echo "  make linux-arm64    Linux aarch64"
	@echo "  make windows-amd64  Windows x86_64"
	@echo "  make windows-arm64  Windows aarch64"
	@echo "Releasing:"
	@echo "  make bump-version BUMP=bugfix|minor|major   Set the next version"
	@echo "  make github-release                         Tag it; Jenkins and npm publish are later"
	@echo "  make clean          Remove build output"
# agents/ is the checked-in source of truth for skills and AGENTS.md, and no tool reads
# it: each tool looks under its own discovery paths. This creates the symlinks that
# bridge them. The links are gitignored, so a fresh clone needs it once, and it is
# idempotent, so re-running costs nothing.
.PHONY: init
init: hooks
	python3 agents/setup.py link
# .git/hooks is not versioned, so a fresh clone commits with nothing checking it
# until this runs. Idempotent, and part of `init` so nobody has to know it exists.
.PHONY: hooks
	git config core.hooksPath .githooks
.PHONY: build
	cargo build
.PHONY: test
	cargo test --all
.PHONY: fmt
	cargo fmt --all
# The counterpart to the sign-in a Bedrock turn does for itself, for testing that path deliberately
# rather than waiting for a token to expire.
# Every profile, because that is the only thing the CLI offers: `aws sso logout` removes every cached
# token and takes no option to narrow it, so `--profile` would scope nothing while reading as though
# it had. One token serves every profile sharing an sso_session, and other tools reading the same
# cache, Claude Code among them, need a fresh `aws sso login` after this.
.PHONY: aws-logout
aws-logout:
	@echo "ending every cached AWS SSO session; sign in again with: aws sso login --profile <name>"
	aws sso logout
# Everything CI enforces, runnable locally before pushing.
# The toolchain check is last so a run that has something to say says it after the results
# rather than in front of them, and it fails rather than warns: this target's whole claim is
# that passing it means CI passes, and on a toolchain several releases behind that is not true.
.PHONY: check
	cargo fmt --all -- --check
	cargo clippy --all-targets --all-features -- -D warnings
	cargo test --all --locked
	@python3 contrib/check-toolchain.py
# Whether clippy here knows the lints CI will fail on. Run by `check`; on its own it costs
# nothing and answers immediately.
.PHONY: check-toolchain
check-toolchain:
	@python3 contrib/check-toolchain.py
# The mechanical half of the spec check: clause numbering, the tests each clause names,
# the paths it governs, the symbols it guards, and the table in the specs README. No model
# is involved, so this is deterministic and belongs in CI. Whether the code actually does
# what a clause says needs a reading of the governed source: run the check-spec skill for
# that half.
.PHONY: check-spec
check-spec:
	python3 agents/skills/check-spec/selftest.py
	python3 agents/skills/check-spec/check-spec.py --mechanical-only
# unverified-clauses.txt, written from the specs. It is the list of clauses nothing pins, and
# check-spec fails while it and the specs disagree, so this is what to run after giving a clause
# a test, or setting one to none.
.PHONY: write-unverified
write-unverified:
	python3 agents/skills/check-spec/check-spec.py --write-unverified
# What each catalog has of the reference, and what it is missing. The build says so too, in a
# warning, but a warning is only printed when the build script actually runs, so a translator
# working through a file learns nothing from a cached build. This always answers.
# The security scan that comments on our pull requests, before pushing rather than
# after. Nothing here configures it: it arrives as an organization-level workflow
# calling brave/security-action, so contrib/check-reviewdog.sh clones that repository
# and drives its reviewdog runners against this checkout, pinning the tool versions
# its action.yml pins. First run downloads opengrep, reviewdog and the rule set into
# ~/.cache; later runs re-use them and take about half a minute.
# The two targets are the action's own two modes. On a pull request it scans what the
# branch changed; on workflow_dispatch it scans everything. A finding here is one the
# bot would post, so the full scan reports plenty that predates any given branch --
# check-reviewdog is the one to run before pushing.
# No model is involved, so both are deterministic.
.PHONY: check-reviewdog
check-reviewdog:
	@contrib/check-reviewdog.sh
.PHONY: check-reviewdog-full
check-reviewdog-full:
	@contrib/check-reviewdog.sh --full
# The npm-lockfile job. The published package is a thin wrapper that downloads the
# release binary, so the lockfile is the whole supply chain surface it has.
.PHONY: check-npm
	npm ci --ignore-scripts
	npm run lint:lockfile
# The dependency policy in deny.toml. CI runs this target rather than cargo-deny's action,
# so the version below is the only one anywhere and a pass here means what it means there.
# Installed under the cache rather than into ~/.cargo/bin, so running this never changes what
# `cargo deny` means anywhere else.
# unmatched-skip is a warning by default: raised here because a skip entry that no longer
# matches is a recorded reason for a duplicate that is no longer there. --locked for the same
# reason every other cargo command here takes it: the answer is about the versions Cargo.lock
# pins, not the ones a resolve on the spot would pick.
CARGO_DENY_VERSION = 0.20.2
CARGO_DENY_ROOT = $(HOME)/.cache/bravebot-deny
.PHONY: check-deps
check-deps:
	@"$(CARGO_DENY_ROOT)/bin/cargo-deny" --version 2>/dev/null | grep -qx "cargo-deny $(CARGO_DENY_VERSION)" || { \
		echo "building cargo-deny $(CARGO_DENY_VERSION) into $(CARGO_DENY_ROOT), which takes a few minutes"; \
		cargo install --quiet --locked cargo-deny@$(CARGO_DENY_VERSION) --root "$(CARGO_DENY_ROOT)"; \
	"$(CARGO_DENY_ROOT)/bin/cargo-deny" --locked check --deny unmatched-skip
# The minimum-toolchain job. Built in a container pinned to the declared MSRV, because
# rustup is not a given here and a Homebrew or distro Rust cannot switch toolchains.
# Catches a feature that only compiles on a newer toolchain than the release build has.
.PHONY: check-msrv
check-msrv:
	docker run --rm --platform linux/amd64 -e BRAVEBOT_ALLOW_UNCONFIGURED_BUILD=1 \
		-v "$(PWD):/src:ro" -w /work rust:$(MSRV)-slim sh -c '\
		cp -r /src/. /work && \
		cargo build --all --locked'
# The windows job. Nothing else compiles the `#[cfg(windows)]` arms of this tree for a check:
# the cross-build produces the shipped binary and lints nothing, and never compiles a test target
# at all. Cross-compiled rather than run on a Windows host, which clippy allows because it stops
# before linking, and in a container for the reason check-msrv is in one: rustup is not a given
# here, and the C compiler ring wants for the target is a package rather than a toolchain
# component.
.PHONY: check-windows
check-windows:
	docker run --rm --platform linux/amd64 -e BRAVEBOT_ALLOW_UNCONFIGURED_BUILD=1 \
		-v "$(PWD):/src:ro" -w /work rust:slim sh -c '\
		cp -r /src/. /work && \
		apt-get update >/dev/null && \
		apt-get install -y --no-install-recommends gcc-mingw-w64-x86-64 >/dev/null && \
		rustup component add clippy >/dev/null && \
		rustup target add x86_64-pc-windows-gnu >/dev/null && \
		cargo clippy --target x86_64-pc-windows-gnu --all-targets --all-features --locked \
			-- -D warnings'
# Everything any CI enforces, in one target: the jobs in ci.yml plus the security
# scan the organization-level workflow runs. Slower than `check` by a lot -- two
# container builds and a scan -- so `check` stays the inner loop and this is the
# before-you-push pass.
.PHONY: check-all
check-all: check check-spec check-npm check-deps check-msrv check-windows check-reviewdog
.PHONY: locales
	@ref=crates/i18n/locales/en-US.ftl; \
	ids() { grep -oE '^[a-z][a-z0-9-]+ =' "$$1" | tr -d ' ='; }; \
	total=$$(ids $$ref | wc -l | tr -d ' '); \
	echo "en-US  $$total messages, the reference"; \
	for f in crates/i18n/locales/*.ftl; do \
		case "$$f" in *en-US.ftl) continue;; esac; \
		tag=$$(basename "$$f" .ftl); \
		have=$$({ ids $$ref; ids "$$f"; } | sort | uniq -d | wc -l | tr -d ' '); \
		echo "$$tag  $$have of $$total"; \
		{ ids $$ref; ids "$$f"; } | sort | uniq -u | sed 's/^/    missing: /'; \
# Runs the same checks on Linux with the current stable toolchain. Worth doing before
# pushing platform-specific code: a macOS host never compiles the Linux backend, and
# clippy gains lints between releases, so both can fail in CI while passing locally.
# The environment the container needs: the build script refuses an unconfigured build
# without the first, which ci.yml sets for CI, and the shell-mode test reads `$$USER` the way
# a terminal would, which a CI runner image provides and a bare container does not. The third
# is set here and must not be set in CI: Docker Desktop's kernel implements no Landlock at
# all, so the sandbox tests would fail rather than skip, while on a CI runner that same
# failure is the report that the Linux half of confinement went unexercised.
# Threads are capped because several turn tests stand up a mock HTTP server on an ephemeral
# port, and at the container's default parallelism enough of them race that a different one
# fails each run. A native runner has the headroom; Docker here does not.
.PHONY: check-linux
check-linux:
	docker run --rm --platform linux/amd64 -e BRAVEBOT_ALLOW_UNCONFIGURED_BUILD=1 -e USER=root \
		-e BRAVEBOT_ALLOW_MISSING_LANDLOCK=1 \
		-v "$(PWD):/src:ro" -w /work rust:slim sh -c '\
		cp -r /src/. /work && \
		rustup component add clippy rustfmt >/dev/null 2>&1 && \
		cargo fmt --all -- --check && \
		cargo clippy --all-targets --all-features -- -D warnings && \
		cargo test --all -- --test-threads=4'
.PHONY: darwin-arm64
darwin-arm64:
	$(call cross-build,$@,aarch64-apple-darwin)
.PHONY: darwin-amd64
darwin-amd64:
	$(call cross-build,$@,x86_64-apple-darwin)
.PHONY: linux-amd64
linux-amd64:
	$(call cross-build,$@,x86_64-unknown-linux-gnu)
.PHONY: linux-arm64
linux-arm64:
	$(call cross-build,$@,aarch64-unknown-linux-gnu)
.PHONY: windows-amd64
windows-amd64:
	$(call cross-build,$@,x86_64-pc-windows-gnu)
.PHONY: windows-arm64
windows-arm64:
	$(call cross-build,$@,aarch64-pc-windows-gnullvm)
.PHONY: all-platforms
all-platforms: darwin-arm64 darwin-amd64 linux-amd64 linux-arm64 windows-amd64 windows-arm64
	@echo "built:"
	@ls -1 dist/
# Symbols are kept during the build because Rust's own strip can corrupt some targets
# under zigbuild, so they are removed here instead.
# rust-objcopy is LLVM-based and handles Mach-O, ELF, and PE alike, so one tool covers
# every target; the per-target GNU strip binaries are not all present in the image.
# The container is asked where its own toolchain is rather than being told: the image is
# published for two architectures and each variant names that directory after its own triple,
# so a written-out path resolves on one release host and not the other, and states the Rust
# version besides. That directory is also LD_LIBRARY_PATH, because rust-objcopy loads the
# libLLVM beside it.
# Run by the publish job after `all-platforms`, and by nothing in this repository, so the
# release carries stripped binaries while a local cross-build keeps its symbols.
.PHONY: strip
	@for f in dist/$(BINARY)-*; do \
		case "$$f" in *.sha256|*SHA256SUMS) continue;; esac; \
		docker run --rm -v "$(PWD)/dist:/dist" -e ASSET="/dist/$$(basename $$f)" \
			ghcr.io/rust-cross/cargo-zigbuild:0.23.0 sh -c '\
			lib=$$(rustc --print sysroot)/lib && \
			host=$$(rustc -vV | sed -n "s/^host: //p") && \
			LD_LIBRARY_PATH=$$lib \
			"$$lib/rustlib/$$host/bin/rust-objcopy" --strip-all "$$ASSET"'; \
	@echo "stripped:"
	@ls -lh dist/ | awk 'NR>1 {print "  " $$9, $$5}'
# Nothing in this repository runs this, and the publish job hashes the signed binaries itself.
# It is the statement of the format those files have to be in: the digest alone, with no filename
# beside it, which is the only thing the npm postinstall accepts. SHA256SUMS is the conventional
# form of the same hashes, for verifying a download by hand.
.PHONY: checksums
	@cd dist && rm -f ./*.sha256 SHA256SUMS && \
	for f in $(BINARY)-*; do \
		shasum -a 256 "$$f" | awk '{print $$1}' > "$$f.sha256"; \
		shasum -a 256 "$$f" >> SHA256SUMS; \
	@echo "wrote dist/SHA256SUMS"
# Edits the four files that state the version and commits exactly those, so no lockfile is
# left behind still naming the old one. It stops there: nothing is pushed and nothing is
# tagged, and the commit is still reviewed before `github-release` will tag it.
.PHONY: bump-version
bump-version:
	@if [ "$(BUMP)" != "bugfix" ] && [ "$(BUMP)" != "minor" ] && [ "$(BUMP)" != "major" ]; then \
		echo "error: BUMP must be one of: bugfix, minor, major"; \
		exit 1; \
	@set -eu; \
	current="$(VERSION)"; \
	if [ -z "$$current" ]; then \
		echo "error: unable to read version from Cargo.toml"; \
		exit 1; \
	if ! git diff --quiet -- $(VERSION_FILES) || ! git diff --cached --quiet -- $(VERSION_FILES); then \
		echo "error: $(VERSION_FILES) already modified; commit or stash that first"; \
		exit 1; \
	major="$${current%%.*}"; rest="$${current#*.}"; \
	minor="$${rest%%.*}"; patch="$${rest#*.}"; \
	case "$(BUMP)" in \
		bugfix) patch="$$((patch + 1))" ;; \
		minor) minor="$$((minor + 1))"; patch=0 ;; \
		major) major="$$((major + 1))"; minor=0; patch=0 ;; \
	next="$$major.$$minor.$$patch"; \
	awk -v v="$$next" ' \
		BEGIN { in_pkg = 0; done = 0 } \
		/^\[/ { in_pkg = ($$0 == "[workspace.package]") } \
		in_pkg && !done && /^version[[:space:]]*=/ { \
			print "version = \"" v "\""; done = 1; next \
		{ print }' Cargo.toml > Cargo.toml.tmp; \
	mv Cargo.toml.tmp Cargo.toml; \
	if [ "$$(sed -nE 's/^version[[:space:]]*=[[:space:]]*"([0-9]+\.[0-9]+\.[0-9]+)".*/\1/p' Cargo.toml | head -n 1)" != "$$next" ]; then \
		echo "error: Cargo.toml version was not rewritten"; \
		exit 1; \
	cargo update --workspace --offline >/dev/null 2>&1 || cargo update --workspace >/dev/null; \
	V="$$next" node -e ' \
const fs = require("node:fs"); \
const pkg = JSON.parse(fs.readFileSync("package.json", "utf8")); \
if (!process.env.V) { throw new Error("version not passed through"); } \
pkg.version = process.env.V; \
fs.writeFileSync("package.json", JSON.stringify(pkg, null, 2) + "\n");'; \
	BRAVEBOT_INSTALL_SKIP_DOWNLOAD=1 npm install --package-lock-only --ignore-scripts >/dev/null; \
	git commit -q -m "Bump version to $$next" -- $(VERSION_FILES); \
	echo "committed: bumped $$current -> $$next ($(VERSION_FILES))"; \
	echo "review it, land it on main, then run: make github-release"
# Tags the current version and pushes it. GitHub Actions runs CI on the tag.
# Signed assets and the npm package are published later, each by hand: Jenkins
# (bravebot-build with UPLOAD and RELEASE), then Actions → Publish npm.
# A refused push takes the tag with it. A tag left behind locally is one the
# remote never got, and the next run reports the version as already tagged
# while the releases page shows nothing.
.PHONY: github-release
github-release:
	@set -eu; \
	if [ -z "$(VERSION)" ]; then \
		echo "error: unable to read version from Cargo.toml"; \
		exit 1; \
	if [ "$$(node -p 'require("./package.json").version')" != "$(VERSION)" ]; then \
		echo "error: package.json version does not match Cargo.toml ($(VERSION)); run make bump-version"; \
		exit 1; \
	if ! git diff --quiet || ! git diff --cached --quiet; then \
		echo "error: working tree must be clean before tagging"; \
		exit 1; \
	branch="$$(git rev-parse --abbrev-ref HEAD)"; \
	if [ "$$branch" != "main" ]; then \
		echo "error: releases are tagged from main, not $$branch"; \
		exit 1; \
	if ! git remote get-url "$(RELEASE_REMOTE)" >/dev/null 2>&1; then \
		echo "error: no remote named $(RELEASE_REMOTE); set bravebot.releaseRemote to one this clone has"; \
		exit 1; \
	git fetch --quiet "$(RELEASE_REMOTE)" main; \
	if [ "$$(git rev-parse HEAD)" != "$$(git rev-parse $(RELEASE_REMOTE)/main)" ]; then \
		echo "error: HEAD differs from $(RELEASE_REMOTE)/main; push or pull first"; \
		exit 1; \
	if git rev-parse -q --verify "refs/tags/$(TAG)" >/dev/null; then \
		echo "error: tag $(TAG) already exists"; \
		exit 1; \
	git tag -a -m "bravebot $(TAG)" "$(TAG)"; \
	if ! git push "$(RELEASE_REMOTE)" "$(TAG)"; then \
		git tag -d "$(TAG)"; \
		echo "error: push failed; removed the local $(TAG) so this can be run again"; \
		exit 1; \
	echo "pushed $(TAG) to $(RELEASE_REMOTE); GitHub Actions will run CI on the tag"; \
	echo "publish signed assets later with Jenkins job bravebot-build (UPLOAD and RELEASE)"; \
	echo "then publish npm: gh workflow run publish-npm.yml --ref $(TAG) -f tag=$(TAG)"; \
	echo "watch CI with: gh run watch --repo brave/bravebot"
.PHONY: clean
	cargo clean
	rm -rf dist
# Configuration reaches the build as a BuildKit secret rather than a build argument,
# which would record the signing key in the image metadata. The temporary file is
# mode 600 and removed even if the build fails.
define cross-build
	env_file="$$(mktemp)"; trap 'rm -f "$$env_file"' EXIT INT TERM; \
	for name in $(BUILD_ENV); do \
		eval "value=\$$$$name"; \
		if [ -n "$$value" ]; then printf 'export %s=%s\n' "$$name" "$$value" >> "$$env_file"; fi; \
	DOCKER_BUILDKIT=1 docker build -f Dockerfile.cross -t $(BINARY)-$(1) \
		--build-arg TARGET=$(2) \
		--secret id=bravebot_env,src="$$env_file" .
	$(call extract,$(BINARY)-$(1),$(1))
# `docker create` on a scratch image needs a command argument even though it never
# runs; the container exists only so the binary can be copied out.
define extract
	mkdir -p dist
	docker rm -f tmp-$(BINARY)-$(2) 2>/dev/null || true
	docker create --name tmp-$(BINARY)-$(2) $(1) /dev/null
	docker cp tmp-$(BINARY)-$(2):/$(BINARY) dist/$(call artifact,$(2))
	docker rm tmp-$(BINARY)-$(2)
define artifact
$(BINARY)-$(1)$(if $(findstring windows,$(1)),.exe,)
Navigation Menu

Uh oh!

Uh oh!

FilesExpand file tree

Makefile

Latest commit

History

Makefile

File metadata and controls
