diff --git a/scripts/setup/AGENT_README.md b/scripts/setup/AGENT_README.md index 00d263285..f7a0abf41 100644 --- a/scripts/setup/AGENT_README.md +++ b/scripts/setup/AGENT_README.md @@ -10,11 +10,12 @@ Installs all system dependencies and downloads required binaries. Safe to re-run **Ubuntu/Debian** (via apt): - Adds the official PostgreSQL apt repo (PGDG) for PG18 -- Installs `postgresql-18`, `redis-server`, `default-jre-headless` +- Adds the official Redis apt repo for Redis Stack (required for RedisJSON) +- Installs `postgresql-18`, `redis-stack-server`, `default-jre-headless` - Installs `clickhouse-server` and `clickhouse-client` from the official ClickHouse apt repo **macOS** (via Homebrew — install brew first if you don't have it): -- Installs `postgresql@18`, `redis`, `clickhouse`, `openjdk` via brew +- Installs `postgresql@18`, `redis-stack`, `clickhouse`, `openjdk` via brew **Both**: - Downloads the ElasticMQ jar to `~/.autumn-agent/elasticmq/elasticmq.jar` and writes its config @@ -55,7 +56,7 @@ Set these in the process environment before running `bun dev:agent` and they wil | Service | Port | Notes | |---|---|---| | PostgreSQL | 5432 | Database: `autumn`, user: `postgres`, password: `postgres` | -| Redis | 6379 | Used for `CACHE_URL` and `CACHE_URL_US_EAST` | +| Redis Stack | 6379 | Used for `CACHE_URL` and `CACHE_URL_US_EAST` (RedisJSON required) | | ElasticMQ | 9324 | Local SQS replacement, queue: `autumn.fifo` | | ClickHouse | 8123 | Used for `TINYBIRD_CLICKHOUSE_URL` | | Server | 8080 | Autumn API server | diff --git a/scripts/setup/agent-bootstrap.sh b/scripts/setup/agent-bootstrap.sh index 66fc844be..57d9934c1 100755 --- a/scripts/setup/agent-bootstrap.sh +++ b/scripts/setup/agent-bootstrap.sh @@ -16,10 +16,15 @@ if [ "$OS" = "Darwin" ]; then BREW_NEEDED=() command -v psql >/dev/null 2>&1 || BREW_NEEDED+=(postgresql@18) - command -v redis-server >/dev/null 2>&1 || BREW_NEEDED+=(redis) + command -v redis-stack-server >/dev/null 2>&1 || BREW_NEEDED+=(redis-stack) command -v clickhouse >/dev/null 2>&1 || BREW_NEEDED+=(clickhouse) command -v java >/dev/null 2>&1 || BREW_NEEDED+=(openjdk) + # redis-stack lives in the redis-stack tap + if [[ " ${BREW_NEEDED[*]} " == *" redis-stack "* ]]; then + brew tap redis-stack/redis-stack >/dev/null 2>&1 || true + fi + if [ ${#BREW_NEEDED[@]} -gt 0 ]; then log "Installing via brew: ${BREW_NEEDED[*]}" brew install "${BREW_NEEDED[@]}" @@ -49,18 +54,34 @@ else if [ ! -f /etc/apt/sources.list.d/pgdg.list ]; then log "Adding official PostgreSQL apt repo (PGDG)" sudo mkdir -p /etc/apt/keyrings - curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ - | sudo gpg --dearmor -o /etc/apt/keyrings/postgresql.gpg + curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc -o /tmp/pgdg.asc + sudo gpg --batch --yes -o /etc/apt/keyrings/postgresql.gpg --dearmor /tmp/pgdg.asc + rm -f /tmp/pgdg.asc echo "deb [signed-by=/etc/apt/keyrings/postgresql.gpg] https://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo $VERSION_CODENAME)-pgdg main" \ | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null sudo apt-get update -qq fi fi + # Redis Stack (provides RedisJSON, required by Autumn's Lua scripts) + # Redis Stack is published for jammy only; that package works cleanly on noble too. + if ! command -v redis-stack-server >/dev/null 2>&1; then + if [ ! -f /etc/apt/sources.list.d/redis.list ]; then + log "Adding Redis Stack apt repo" + sudo mkdir -p /etc/apt/keyrings + curl -fsSL https://packages.redis.io/gpg -o /tmp/redis-stack.asc + sudo gpg --batch --yes -o /etc/apt/keyrings/redis-archive-keyring.gpg --dearmor /tmp/redis-stack.asc + rm -f /tmp/redis-stack.asc + echo "deb [signed-by=/etc/apt/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb jammy main" \ + | sudo tee /etc/apt/sources.list.d/redis.list >/dev/null + sudo apt-get update -qq + fi + fi + APT_NEEDED=() - command -v pg_ctlcluster >/dev/null 2>&1 || APT_NEEDED+=(postgresql-18) - command -v redis-server >/dev/null 2>&1 || APT_NEEDED+=(redis-server) - command -v java >/dev/null 2>&1 || APT_NEEDED+=(default-jre-headless) + command -v pg_ctlcluster >/dev/null 2>&1 || APT_NEEDED+=(postgresql-18) + command -v redis-stack-server >/dev/null 2>&1 || APT_NEEDED+=(redis-stack-server) + command -v java >/dev/null 2>&1 || APT_NEEDED+=(default-jre-headless) if [ ${#APT_NEEDED[@]} -gt 0 ]; then log "Installing via apt: ${APT_NEEDED[*]}" @@ -71,8 +92,9 @@ else if ! command -v clickhouse-server >/dev/null 2>&1; then log "Installing ClickHouse from official apt repo" sudo mkdir -p /etc/apt/keyrings - curl -fsSL 'https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key' \ - | sudo gpg --dearmor -o /etc/apt/keyrings/clickhouse.gpg + curl -fsSL 'https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key' -o /tmp/clickhouse.asc + sudo gpg --batch --yes -o /etc/apt/keyrings/clickhouse.gpg --dearmor /tmp/clickhouse.asc + rm -f /tmp/clickhouse.asc echo 'deb [signed-by=/etc/apt/keyrings/clickhouse.gpg] https://packages.clickhouse.com/deb stable main' \ | sudo tee /etc/apt/sources.list.d/clickhouse.list >/dev/null sudo apt-get update -qq diff --git a/scripts/setup/agent-services.sh b/scripts/setup/agent-services.sh index 304905636..d3bb0b233 100755 --- a/scripts/setup/agent-services.sh +++ b/scripts/setup/agent-services.sh @@ -12,12 +12,29 @@ log "Starting system services" if [ "$OS" = "Darwin" ]; then brew services start postgresql@18 >/dev/null 2>&1 || true - brew services start redis >/dev/null 2>&1 || true + brew services start redis-stack >/dev/null 2>&1 || true brew services start clickhouse >/dev/null 2>&1 || true else - sudo service postgresql start >/dev/null 2>&1 || true - sudo service redis-server start >/dev/null 2>&1 || true - sudo service clickhouse-server start >/dev/null 2>&1 || true + sudo service postgresql start >/dev/null 2>&1 || true + sudo service clickhouse-server start >/dev/null 2>&1 || true + + # redis-stack-server ships a systemd unit but no SysV init script. + # Try `service` first (works when systemd is the init), then fall back + # to invoking the binary directly (works in containers without systemd). + if ! redis-cli -p 6379 PING >/dev/null 2>&1; then + if ! sudo service redis-stack-server start >/dev/null 2>&1; then + REDIS_LOG_DIR="${HOME}/.autumn-agent/logs" + mkdir -p "$REDIS_LOG_DIR" + log "Starting redis-stack-server manually" + nohup /opt/redis-stack/bin/redis-stack-server \ + >"$REDIS_LOG_DIR/redis-stack.log" 2>&1 & + disown || true + for i in $(seq 1 30); do + redis-cli -p 6379 PING >/dev/null 2>&1 && break + sleep 0.2 + done + fi + fi fi # =============================================================