diff --git a/package.json b/package.json index 3b450c708..2bb143b99 100644 --- a/package.json +++ b/package.json @@ -80,6 +80,8 @@ "setup": "node scripts/setup/setup.js", "setup:s3-admin": "bun scripts/setup/setupS3Admin.ts", "setup:test": "infisical run --env=dev -- bun scripts/setup/setup-test.ts", + "agent:bootstrap": "bash scripts/setup/agent-bootstrap.sh", + "dev:agent": "bash scripts/setup/agent-services.sh && bun scripts/dev.ts", "migrate-functions": "infisical run --env=dev -- bun scripts/migrations/migrate-functions.ts", "migrate-functions:test": "infisical run --env=test -- bun scripts/migrations/migrate-functions.ts", "migrate-functions:prod": "infisical run --env=prod -- bun scripts/migrations/migrate-functions.ts", diff --git a/scripts/setup/AGENT_README.md b/scripts/setup/AGENT_README.md new file mode 100644 index 000000000..be9148216 --- /dev/null +++ b/scripts/setup/AGENT_README.md @@ -0,0 +1,74 @@ +# Agent Dev Setup + +Two commands to spin up the full Autumn stack on a fresh Ubuntu VM — no Docker, no Infisical, no Supabase cloud signup, no interactive prompts. + +## Commands + +### `bun agent:bootstrap` — one-time + +Installs all system dependencies and downloads required binaries. Safe to re-run; every step is guarded and completes in ~1s if already installed. + +- Installs `postgresql-16` and `redis-server` via apt +- Installs `clickhouse-server` and `clickhouse-client` from the official ClickHouse apt repo +- Downloads the ElasticMQ jar to `/opt/elasticmq/elasticmq.jar` and writes its config +- Runs `bun install --frozen-lockfile` if `node_modules` is missing + +### `bun dev:agent` — every session + +Starts all local services, creates the database, writes env files, runs migrations, then starts the dev servers. + +1. Starts `postgresql`, `redis-server`, and `clickhouse-server` via `service` +2. Starts ElasticMQ in the background on `:9324` (skipped if already running) +3. Creates the `autumn` Postgres database if it does not exist +4. Writes `server/.env` (skips if already present) and `vite/.env` from `vite/.env.example` +5. Runs `bun db:migrate` +6. Launches server `:8080`, vite `:3000`, checkout `:3001`, and workers via `scripts/dev.ts` + +## Static team-wide env vars + +Set these in the process environment before running `bun dev:agent` and they will be written into `server/.env` on first run: + +| Variable | Description | +|---|---| +| `STRIPE_SANDBOX_SECRET_KEY` | Stripe test-mode secret key | +| `STRIPE_SANDBOX_WEBHOOK_SECRET` | Stripe test-mode webhook signing secret | +| `STRIPE_SANDBOX_CLIENT_ID` | Stripe Connect test platform client ID | +| `STRIPE_LIVE_SECRET_KEY` | Stripe live secret key | +| `STRIPE_LIVE_WEBHOOK_SECRET` | Stripe live webhook signing secret | +| `STRIPE_LIVE_CLIENT_ID` | Stripe Connect live platform client ID | +| `ANTHROPIC_API_KEY` | Anthropic API key | +| `RESEND_API_KEY` | Resend email API key | +| `RESEND_DOMAIN` | Resend sending domain | +| `SVIX_API_KEY` | Svix webhook API key | +| `POSTHOG_API_KEY` | PostHog project API key | +| `POSTHOG_HOST` | PostHog instance URL | + +## Local services and ports + +| Service | Port | Notes | +|---|---|---| +| PostgreSQL | 5432 | Database: `autumn`, user: `postgres`, password: `postgres` | +| Redis | 6379 | Used for `CACHE_URL` and `CACHE_URL_US_EAST` | +| ElasticMQ | 9324 | Local SQS replacement, queue: `autumn.fifo` | +| ClickHouse | 8123 | Used for `TINYBIRD_CLICKHOUSE_URL` | +| Server | 8080 | Autumn API server | +| Vite | 3000 | Frontend dev server | +| Checkout | 3001 | Checkout app dev server | + +## SQS isolation + +Each agent instance gets its own local ElasticMQ. `SQS_QUEUE_URL` is always hardcoded to `http://localhost:9324/000000000000/autumn.fifo` in `server/.env` — the shared `SQS_QUEUE_URL` from the Capy environment is intentionally ignored to prevent agents from consuming a shared queue. + +## Known limitations + +- **EventBridge lock-receipt scheduler**: `createSchedule` and `deleteSchedule` are no-ops when running against a local (non-amazonaws.com) queue URL. Lock-receipt expiry will not fire automatically in local dev. +- **Supabase logo upload**: Storage is not configured locally. Logo upload endpoints return 400 — expected. +- **Stripe webhooks**: `STRIPE_WEBHOOK_URL=http://localhost:8080`. Configure a Stripe CLI webhook forwarder separately if you need live webhook testing. + +## Reset + +To regenerate `server/.env` with fresh secrets: + +```bash +rm server/.env vite/.env && bun dev:agent +``` diff --git a/scripts/setup/agent-bootstrap.sh b/scripts/setup/agent-bootstrap.sh new file mode 100755 index 000000000..708f402d7 --- /dev/null +++ b/scripts/setup/agent-bootstrap.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -euo pipefail + +log() { echo "[agent-bootstrap] $*"; } + +# --- 1. postgresql-16 and redis-server via apt --- +APT_NEEDED=() +command -v pg_ctlcluster >/dev/null 2>&1 || APT_NEEDED+=(postgresql-16) +command -v redis-server >/dev/null 2>&1 || APT_NEEDED+=(redis-server) + +if [ ${#APT_NEEDED[@]} -gt 0 ]; then + log "Installing system packages: ${APT_NEEDED[*]}" + sudo apt-get update -qq + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq "${APT_NEEDED[@]}" +fi + +# --- 2. ClickHouse from official apt repo --- +if ! command -v clickhouse-server >/dev/null 2>&1; then + log "Installing ClickHouse" + sudo mkdir -p /etc/apt/keyrings + curl -fsSL 'https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key' \ + | sudo gpg --dearmor -o /etc/apt/keyrings/clickhouse.gpg + echo 'deb [signed-by=/etc/apt/keyrings/clickhouse.gpg] https://packages.clickhouse.com/deb stable main' \ + | sudo tee /etc/apt/sources.list.d/clickhouse.list >/dev/null + sudo apt-get update -qq + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \ + -o Dpkg::Options::='--force-confnew' \ + clickhouse-server clickhouse-client +fi + +# --- 3. ElasticMQ jar (local SQS-compatible queue, no Docker) --- +ELASTICMQ_VERSION="1.6.11" +ELASTICMQ_JAR="/opt/elasticmq/elasticmq.jar" +if [ ! -f "$ELASTICMQ_JAR" ]; then + log "Downloading ElasticMQ $ELASTICMQ_VERSION" + sudo mkdir -p /opt/elasticmq + sudo curl -fsSL -o "$ELASTICMQ_JAR" \ + "https://s3-eu-west-1.amazonaws.com/softwaremill-public/elasticmq-server-${ELASTICMQ_VERSION}.jar" +fi + +# ElasticMQ config: single FIFO queue 'autumn.fifo' +ELASTICMQ_CONF="/opt/elasticmq/elasticmq.conf" +if [ ! -f "$ELASTICMQ_CONF" ]; then + sudo tee "$ELASTICMQ_CONF" >/dev/null <<'EOF' +include classpath("application.conf") +node-address { + protocol = http + host = "localhost" + port = 9324 + context-path = "" +} +rest-sqs { + enabled = true + bind-port = 9324 + bind-hostname = "0.0.0.0" + sqs-limits = strict +} +queues { + "autumn.fifo" { + defaultVisibilityTimeout = 30 seconds + receiveMessageWait = 0 seconds + fifo = true + contentBasedDeduplication = true + } +} +EOF +fi + +# --- 4. Bun workspace install --- +if [ ! -d node_modules ]; then + log "Installing workspace dependencies" + bun install --frozen-lockfile +fi + +log "Bootstrap complete. Run: bun dev:agent" diff --git a/scripts/setup/agent-services.sh b/scripts/setup/agent-services.sh new file mode 100755 index 000000000..8f954470f --- /dev/null +++ b/scripts/setup/agent-services.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +log() { echo "[agent-services] $*"; } + +# --- 1. Start postgresql, redis, clickhouse --- +log "Starting system services" +sudo service postgresql start >/dev/null 2>&1 || true +sudo service redis-server start >/dev/null 2>&1 || true +sudo service clickhouse-server start >/dev/null 2>&1 || true + +# --- 2. ElasticMQ --- +if ! pgrep -f 'elasticmq.*\.jar' >/dev/null 2>&1; then + log "Starting ElasticMQ on :9324" + sudo mkdir -p /var/log/autumn && sudo chmod 0777 /var/log/autumn + nohup java \ + -Dconfig.file=/opt/elasticmq/elasticmq.conf \ + -jar /opt/elasticmq/elasticmq.jar \ + >/var/log/autumn/elasticmq.log 2>&1 & + disown || true + log "Waiting for ElasticMQ to be ready" + for i in $(seq 1 30); do + curl -sf -o /dev/null 'http://localhost:9324/?Action=ListQueues&Version=2012-11-05' && break + sleep 0.5 + done +fi + +# --- 3. Ensure Postgres DB exists --- +DB_NAME="autumn" +sudo -u postgres psql -tc "SELECT 1 FROM pg_database WHERE datname = '$DB_NAME'" 2>/dev/null \ + | grep -q 1 || sudo -u postgres createdb "$DB_NAME" +sudo -u postgres psql -c "ALTER USER postgres WITH PASSWORD 'postgres';" >/dev/null 2>&1 || true + +# --- 4. Write env files (no-op if server/.env already exists) --- +bun scripts/setup/writeAgentEnv.ts + +# --- 5. DB migrations --- +log "Running migrations" +bun db:generate >/dev/null 2>&1 || true +bun db:migrate + +log "All services ready" diff --git a/scripts/setup/writeAgentEnv.ts b/scripts/setup/writeAgentEnv.ts new file mode 100644 index 000000000..804ec9be9 --- /dev/null +++ b/scripts/setup/writeAgentEnv.ts @@ -0,0 +1,83 @@ +import { randomBytes } from "node:crypto"; +import { copyFileSync, existsSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +const root = join(import.meta.dir, "..", ".."); +const serverEnvPath = join(root, "server", ".env"); +const viteEnvPath = join(root, "vite", ".env"); +const viteExamplePath = join(root, "vite", ".env.example"); + +const genUrlSafeBase64 = ({ bytes }: { bytes: number }): string => + randomBytes(bytes) + .toString("base64") + .replace(/\+/g, "-") + .replace(/\//g, "_") + .replace(/=+$/g, ""); + +if (existsSync(serverEnvPath)) { + console.log("[writeAgentEnv] server/.env already exists — skipping generation"); +} else { + const passThrough = [ + "STRIPE_SANDBOX_CLIENT_ID", + "STRIPE_SANDBOX_SECRET_KEY", + "STRIPE_SANDBOX_WEBHOOK_SECRET", + "STRIPE_LIVE_CLIENT_ID", + "STRIPE_LIVE_SECRET_KEY", + "STRIPE_LIVE_WEBHOOK_SECRET", + "ANTHROPIC_API_KEY", + "RESEND_API_KEY", + "RESEND_DOMAIN", + "SVIX_API_KEY", + "POSTHOG_API_KEY", + "POSTHOG_HOST", + ] as const; + + const passLines = passThrough + .map((k) => `${k}=${process.env[k] ?? ""}`) + .join("\n"); + + const content = `# ----------------------------------------------------------------------- +# Generated by scripts/setup/writeAgentEnv.ts +# Delete this file and re-run bun dev:agent to regenerate. +# ----------------------------------------------------------------------- + +# Per-agent random secrets +BETTER_AUTH_SECRET=${genUrlSafeBase64({ bytes: 64 })} +ENCRYPTION_IV=${genUrlSafeBase64({ bytes: 16 })} +ENCRYPTION_PASSWORD=${genUrlSafeBase64({ bytes: 64 })} + +# Local services +DATABASE_URL=postgresql://postgres:postgres@localhost:5432/autumn +CACHE_URL=redis://localhost:6379 +CACHE_URL_US_EAST=redis://localhost:6379 +REDIS_URL=redis://localhost:6379 + +# ElasticMQ (local SQS, per-agent isolated queue) +SQS_QUEUE_URL=http://localhost:9324/000000000000/autumn.fifo +AWS_REGION=us-east-1 +AWS_ACCESS_KEY_ID=x +AWS_SECRET_ACCESS_KEY=x + +# ClickHouse (local) +TINYBIRD_CLICKHOUSE_URL=http://localhost:8123 + +# App URLs +BETTER_AUTH_URL=http://localhost:8080 +CLIENT_URL=http://localhost:3000 +STRIPE_WEBHOOK_URL=http://localhost:8080 + +# Static team-wide (pass-through from process.env) +${passLines} + +# Environment +NODE_ENV=development +`; + + writeFileSync(serverEnvPath, content); + console.log("[writeAgentEnv] Wrote server/.env"); +} + +if (!existsSync(viteEnvPath) && existsSync(viteExamplePath)) { + copyFileSync(viteExamplePath, viteEnvPath); + console.log("[writeAgentEnv] Wrote vite/.env from .env.example"); +} diff --git a/server/src/external/aws/eventbridge/eventBridgeUtils.ts b/server/src/external/aws/eventbridge/eventBridgeUtils.ts index 8afc29bfa..fe4e5a186 100644 --- a/server/src/external/aws/eventbridge/eventBridgeUtils.ts +++ b/server/src/external/aws/eventbridge/eventBridgeUtils.ts @@ -4,8 +4,12 @@ import { ResourceNotFoundException, } from "@aws-sdk/client-scheduler"; import { logger } from "@/external/logtail/logtailUtils.js"; +import { extractLocalEndpoint } from "@/queue/initSqs.js"; import { schedulerClient } from "./initEventBridge.js"; +const isLocalQueue = (): boolean => + !!extractLocalEndpoint({ queueUrl: process.env.SQS_QUEUE_URL }); + const SCHEDULE_GROUP = "default"; const SCHEDULER_ROLE_ARN = process.env.AWS_EVENTBRIDGE_SCHEDULER_ROLE_ARN || ""; @@ -33,6 +37,12 @@ export const createSchedule = async ({ sqsMessageBody: string; messageGroupId: string; }) => { + if (isLocalQueue()) { + logger.debug( + "[EventBridge] createSchedule skipped (local SQS queue — no EventBridge in dev)", + ); + return; + } // EventBridge at-expression: at(yyyy-mm-ddThh:mm:ss) const pad = (n: number) => String(n).padStart(2, "0"); const d = scheduleAt; @@ -70,6 +80,12 @@ export const deleteSchedule = async ({ }: { scheduleName: string; }) => { + if (isLocalQueue()) { + logger.debug( + "[EventBridge] deleteSchedule skipped (local SQS queue — no EventBridge in dev)", + ); + return; + } try { await schedulerClient.send( new DeleteScheduleCommand({ diff --git a/server/src/queue/initSqs.ts b/server/src/queue/initSqs.ts index 369cf6580..ff4828740 100644 --- a/server/src/queue/initSqs.ts +++ b/server/src/queue/initSqs.ts @@ -6,16 +6,35 @@ import { // ============ FIFO Queue (primary) ============ -const getSqsClientConfig = () => ({ - region: - extractRegionFromQueueUrl({ - queueUrl: process.env.SQS_QUEUE_URL, - }) || DEFAULT_AWS_REGION, - credentials: { - accessKeyId: process.env.AWS_ACCESS_KEY_ID || "", - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || "", - }, -}); +/** Returns a base endpoint URL if the queue URL points to a non-AWS host (e.g. ElasticMQ). */ +export const extractLocalEndpoint = ({ + queueUrl, +}: { + queueUrl: string | undefined; +}): string | undefined => { + if (!queueUrl) return undefined; + try { + const url = new URL(queueUrl); + if (url.hostname.endsWith("amazonaws.com")) return undefined; + return `${url.protocol}//${url.host}`; + } catch { + return undefined; + } +}; + +const getSqsClientConfig = () => { + const queueUrl = process.env.SQS_QUEUE_URL; + const endpoint = extractLocalEndpoint({ queueUrl }); + return { + region: + extractRegionFromQueueUrl({ queueUrl }) || DEFAULT_AWS_REGION, + ...(endpoint ? { endpoint } : {}), + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID || "", + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || "", + }, + }; +}; let sqsClient = new SQSClient(getSqsClientConfig());