diff --git a/.github/actions/docker-setup/action.yaml b/.github/actions/docker-setup/action.yaml new file mode 100644 index 0000000..d07ec5a --- /dev/null +++ b/.github/actions/docker-setup/action.yaml @@ -0,0 +1,31 @@ +name: 'Docker Setup' +description: 'Set up Docker Buildx and log in to Docker Hub' +inputs: + docker_username: + description: 'Docker Hub username' + required: true + docker_password: + description: 'Docker Hub password' + required: true + github_token: + description: 'GitHub token' + required: true +runs: + using: 'composite' + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ inputs.docker_username }} + password: ${{ inputs.docker_password }} + + # This will be used as a secret to authenticate with Git repo pulls + - name: Create .netrc file + run: | + echo "machine github.com" > ${{ runner.temp }}/netrc + echo "login x-access-token" >> ${{ runner.temp }}/netrc + echo "password ${{ inputs.github_token }}" >> ${{ runner.temp }}/netrc + shell: bash diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 34199ee..2593a43 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -4,26 +4,37 @@ on: workflow_dispatch: inputs: version: - description: "Version (e.g. 0.1.0 or v0.1.0)" + description: 'Version' required: true type: string latest: - description: "Latest" + description: 'Latest' required: true type: boolean default: true + reuse_engine_version: + description: 'Reuse artifacts from this version (skips building)' + required: false + type: string defaults: run: + # Enable fail-fast behavior shell: bash -e {0} env: + # Disable incremental compilation for faster from-scratch builds CARGO_INCREMENTAL: 0 jobs: setup: name: "Setup" runs-on: ubuntu-24.04 + permissions: + # Allow pushing to GitHub + contents: write + # Allows authentication + id-token: write steps: - uses: actions/checkout@v4 with: @@ -31,89 +42,160 @@ jobs: - uses: dtolnay/rust-toolchain@stable - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 with: node-version: 20 - cache: pnpm - - name: Install tsx - run: npm install -g tsx + - run: corepack enable - - name: Install workspace dependencies - run: pnpm install --no-frozen-lockfile - - - name: Install release script dependencies + - name: Setup + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + R2_RELEASES_ACCESS_KEY_ID: ${{ secrets.R2_RELEASES_ACCESS_KEY_ID }} + R2_RELEASES_SECRET_ACCESS_KEY: ${{ secrets.R2_RELEASES_SECRET_ACCESS_KEY }} run: | - cd scripts/release - pnpm install --no-frozen-lockfile --ignore-workspace + # Configure Git + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + # Authenticate with NPM + cat << EOF > ~/.npmrc + //registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }} + EOF + + # Install dependencies + pnpm install + + # Install tsx globally + npm install -g tsx + + # Build command based on inputs + CMD="./scripts/release/main.ts --version \"${{ github.event.inputs.version }}\" --phase setup-ci" - - name: Run setup phase - run: | - CMD="./scripts/release/main.ts --version '${{ inputs.version }}' --phase setup-ci" if [ "${{ inputs.latest }}" != "true" ]; then CMD="$CMD --no-latest" fi + + if [ -n "${{ inputs.reuse_engine_version }}" ]; then + CMD="$CMD --reuse-engine-version \"${{ inputs.reuse_engine_version }}\"" + fi + eval "$CMD" binaries: - name: "Build Binaries" + name: "Build & Upload Binaries" needs: [setup] + if: ${{ !inputs.reuse_engine_version }} strategy: matrix: include: - platform: linux + runner: depot-ubuntu-24.04-8 target: x86_64-unknown-linux-musl binary_ext: "" arch: x86_64 - platform: windows + runner: depot-ubuntu-24.04-8 target: x86_64-pc-windows-gnu binary_ext: ".exe" arch: x86_64 - platform: macos + runner: depot-ubuntu-24.04-8 target: x86_64-apple-darwin binary_ext: "" arch: x86_64 - platform: macos + runner: depot-ubuntu-24.04-8 target: aarch64-apple-darwin binary_ext: "" arch: aarch64 - runs-on: ubuntu-24.04 + runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: pnpm/action-setup@v4 - - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: pnpm - - - name: Build inspector frontend - run: | - pnpm install --no-frozen-lockfile - SANDBOX_AGENT_SKIP_INSPECTOR=1 pnpm --filter @sandbox-agent/inspector build - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build binary run: | + # Use Docker BuildKit + export DOCKER_BUILDKIT=1 + + # Build the binary using our Dockerfile docker/release/build.sh ${{ matrix.target }} + + # Make sure dist directory exists and binary is there ls -la dist/ - - name: Upload artifact - uses: actions/upload-artifact@v4 + - name: Upload to R2 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.R2_RELEASES_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_RELEASES_SECRET_ACCESS_KEY }} + run: | + # Install dependencies for AWS CLI + sudo apt-get update + sudo apt-get install -y unzip curl + + # Install AWS CLI + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + sudo ./aws/install --update + + COMMIT_SHA_SHORT="${GITHUB_SHA::7}" + BINARY_PATH="dist/sandbox-agent-${{ matrix.target }}${{ matrix.binary_ext }}" + + # Must specify --checksum-algorithm for compatibility with R2 + aws s3 cp \ + "${BINARY_PATH}" \ + "s3://rivet-releases/sandbox-agent/${COMMIT_SHA_SHORT}/binaries/sandbox-agent-${{ matrix.target }}${{ matrix.binary_ext }}" \ + --region auto \ + --endpoint-url https://2a94c6a0ced8d35ea63cddc86c2681e7.r2.cloudflarestorage.com \ + --checksum-algorithm CRC32 + + docker: + name: "Build & Push Docker Images" + needs: [setup] + if: ${{ !inputs.reuse_engine_version }} + strategy: + matrix: + include: + - platform: linux/arm64 + runner: depot-ubuntu-24.04-arm-8 + arch_suffix: -arm64 + - platform: linux/amd64 + runner: depot-ubuntu-24.04-8 + arch_suffix: -amd64 + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 with: - name: binary-${{ matrix.target }} - path: dist/sandbox-agent-${{ matrix.target }}${{ matrix.binary_ext }} + fetch-depth: 0 + + - name: Set outputs + id: vars + run: echo "sha_short=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT + + - uses: ./.github/actions/docker-setup + with: + docker_username: ${{ secrets.DOCKER_CI_USERNAME }} + docker_password: ${{ secrets.DOCKER_CI_ACCESS_TOKEN }} + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Build & Push + uses: docker/build-push-action@v4 + with: + context: . + push: true + tags: rivetdev/sandbox-agent:${{ steps.vars.outputs.sha_short }}${{ matrix.arch_suffix }} + file: docker/runtime/Dockerfile + platforms: ${{ matrix.platform }} complete: name: "Complete" - needs: [setup, binaries] + needs: [setup, docker, binaries] + if: ${{ always() && !cancelled() && needs.setup.result == 'success' && (needs.docker.result == 'success' || needs.docker.result == 'skipped') && (needs.binaries.result == 'success' || needs.binaries.result == 'skipped') }} runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 @@ -122,53 +204,47 @@ jobs: - uses: dtolnay/rust-toolchain@stable - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 with: node-version: 20 registry-url: "https://registry.npmjs.org" - cache: pnpm - - name: Install tsx - run: npm install -g tsx + - run: corepack enable - - name: Install workspace dependencies - run: pnpm install --no-frozen-lockfile - - - name: Install release script dependencies - run: | - cd scripts/release - pnpm install --no-frozen-lockfile --ignore-workspace - - - name: Install AWS CLI - run: | - sudo apt-get update - sudo apt-get install -y unzip curl - - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip awscliv2.zip - sudo ./aws/install --update - - - name: Download binaries - uses: actions/download-artifact@v4 + - uses: ./.github/actions/docker-setup with: - path: dist/ - pattern: binary-* - merge-multiple: true + docker_username: ${{ secrets.DOCKER_CI_USERNAME }} + docker_password: ${{ secrets.DOCKER_CI_ACCESS_TOKEN }} + github_token: ${{ secrets.GITHUB_TOKEN }} - - name: List downloaded binaries - run: ls -la dist/ - - - name: Publish & upload artifacts + - name: Complete env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} R2_RELEASES_ACCESS_KEY_ID: ${{ secrets.R2_RELEASES_ACCESS_KEY_ID }} R2_RELEASES_SECRET_ACCESS_KEY: ${{ secrets.R2_RELEASES_SECRET_ACCESS_KEY }} run: | - CMD="./scripts/release/main.ts --version '${{ inputs.version }}' --phase complete-ci --no-validate-git" + # Authenticate with NPM + cat << EOF > ~/.npmrc + //registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }} + EOF + + # Install dependencies + pnpm install + + # Install tsx globally + npm install -g tsx + + # Build command based on inputs + CMD="./scripts/release/main.ts --version \"${{ github.event.inputs.version }}\" --phase complete-ci --no-validate-git" + if [ "${{ inputs.latest }}" != "true" ]; then CMD="$CMD --no-latest" fi + + if [ -n "${{ inputs.reuse_engine_version }}" ]; then + CMD="$CMD --reuse-engine-version \"${{ inputs.reuse_engine_version }}\"" + fi + eval "$CMD" diff --git a/docker/runtime/Dockerfile b/docker/runtime/Dockerfile new file mode 100644 index 0000000..0e71c2f --- /dev/null +++ b/docker/runtime/Dockerfile @@ -0,0 +1,51 @@ +# syntax=docker/dockerfile:1.10.0 + +# Build stage - compile the binary +FROM rust:1.88.0 AS builder + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y \ + musl-tools \ + musl-dev \ + pkg-config \ + ca-certificates \ + git && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN rustup target add x86_64-unknown-linux-musl + +WORKDIR /build +COPY . . + +# Build static binary +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + --mount=type=cache,target=/build/target \ + SANDBOX_AGENT_SKIP_INSPECTOR=1 \ + RUSTFLAGS="-C target-feature=+crt-static" \ + cargo build -p sandbox-agent --release --target x86_64-unknown-linux-musl && \ + cp target/x86_64-unknown-linux-musl/release/sandbox-agent /sandbox-agent + +# Runtime stage - minimal image +FROM debian:bookworm-slim + +RUN apt-get update && apt-get install -y \ + ca-certificates \ + curl \ + git && \ + rm -rf /var/lib/apt/lists/* + +# Copy the binary from builder +COPY --from=builder /sandbox-agent /usr/local/bin/sandbox-agent +RUN chmod +x /usr/local/bin/sandbox-agent + +# Create non-root user +RUN useradd -m -s /bin/bash sandbox +USER sandbox +WORKDIR /home/sandbox + +EXPOSE 2468 + +ENTRYPOINT ["sandbox-agent"] +CMD ["--host", "0.0.0.0", "--port", "2468"] diff --git a/server/packages/agent-credentials/Cargo.toml b/server/packages/agent-credentials/Cargo.toml index ef4e566..d997149 100644 --- a/server/packages/agent-credentials/Cargo.toml +++ b/server/packages/agent-credentials/Cargo.toml @@ -4,6 +4,8 @@ version.workspace = true edition.workspace = true authors.workspace = true license.workspace = true +description.workspace = true +repository.workspace = true [dependencies] serde.workspace = true diff --git a/server/packages/agent-management/Cargo.toml b/server/packages/agent-management/Cargo.toml index d31ed26..18fb5df 100644 --- a/server/packages/agent-management/Cargo.toml +++ b/server/packages/agent-management/Cargo.toml @@ -4,6 +4,8 @@ version.workspace = true edition.workspace = true authors.workspace = true license.workspace = true +description.workspace = true +repository.workspace = true [dependencies] sandbox-agent-agent-credentials.workspace = true diff --git a/server/packages/error/Cargo.toml b/server/packages/error/Cargo.toml index 77a8265..b63e3e0 100644 --- a/server/packages/error/Cargo.toml +++ b/server/packages/error/Cargo.toml @@ -4,6 +4,8 @@ version.workspace = true edition.workspace = true authors.workspace = true license.workspace = true +description.workspace = true +repository.workspace = true [dependencies] serde.workspace = true diff --git a/server/packages/extracted-agent-schemas/Cargo.toml b/server/packages/extracted-agent-schemas/Cargo.toml index 8534627..afc8149 100644 --- a/server/packages/extracted-agent-schemas/Cargo.toml +++ b/server/packages/extracted-agent-schemas/Cargo.toml @@ -4,6 +4,8 @@ version.workspace = true edition.workspace = true authors.workspace = true license.workspace = true +description.workspace = true +repository.workspace = true [dependencies] serde.workspace = true diff --git a/server/packages/sandbox-agent/Cargo.toml b/server/packages/sandbox-agent/Cargo.toml index da2e88f..061337e 100644 --- a/server/packages/sandbox-agent/Cargo.toml +++ b/server/packages/sandbox-agent/Cargo.toml @@ -4,6 +4,8 @@ version.workspace = true edition.workspace = true authors.workspace = true license.workspace = true +description.workspace = true +repository.workspace = true [[bin]] name = "sandbox-agent" diff --git a/server/packages/universal-agent-schema/Cargo.toml b/server/packages/universal-agent-schema/Cargo.toml index 1f9e365..c4f1563 100644 --- a/server/packages/universal-agent-schema/Cargo.toml +++ b/server/packages/universal-agent-schema/Cargo.toml @@ -4,6 +4,8 @@ version.workspace = true edition.workspace = true authors.workspace = true license.workspace = true +description.workspace = true +repository.workspace = true [dependencies] sandbox-agent-extracted-agent-schemas.workspace = true