diff --git a/ai b/ai index 0d561b174..794c164ae 160000 --- a/ai +++ b/ai @@ -1 +1 @@ -Subproject commit 0d561b1747e8f47190a01d7a9bff7d8fcb42c9dd +Subproject commit 794c164aed6e15cde9dc859b2e256c72a02af05d diff --git a/scripts/db/README.md b/scripts/db/README.md index 167e2ae82..7c5f6a2be 100644 --- a/scripts/db/README.md +++ b/scripts/db/README.md @@ -13,7 +13,9 @@ bun db mark-applied [--env=dev|staging|prod] # seed drizzle.__drizzle_migration bun db rebase # auto-resolve a local migration that collided with origin/dev ``` -`migrate` and `migrate:dry` also run a safety check that **refuses to apply any pending migration containing `CREATE INDEX`, `DROP INDEX`, or `REINDEX` without `CONCURRENTLY`**. Those DDL statements take an ACCESS EXCLUSIVE lock and can block reads/writes on busy tables. To get through the check: either rewrite the SQL with `CONCURRENTLY` and apply manually + `mark-applied`, or add `.concurrently()` to the index in your schema and regenerate. +`migrate` applies pending migrations directly via `pg` (using drizzle's own `readMigrationFiles` for parsing + hashing), so the tracking table stays compatible with drizzle and `mark-applied`. Unlike drizzle's built-in `migrate()` — which wraps every migration in a single transaction — our executor runs any statement containing `CONCURRENTLY` in autocommit, so `CREATE INDEX CONCURRENTLY` migrations apply normally. Everything else still runs in a per-migration transaction. + +`migrate` and `migrate:dry` also run a safety check that **refuses to apply any pending migration containing `CREATE INDEX`, `DROP INDEX`, or `REINDEX` without `CONCURRENTLY`**. Those DDL statements take an ACCESS EXCLUSIVE lock and can block reads/writes on busy tables. To get through the check, make the index concurrent: rewrite the SQL with `CONCURRENTLY`, or add `.concurrently()` to the index in your schema and regenerate. Concurrent index migrations then apply through `bun db migrate` with no manual step. `--env` defaults to `dev`. `generate` and `rebase` never touch a DB and don't take `--env`. @@ -133,17 +135,20 @@ scripts/db/ ├── commands/ │ ├── help.ts │ ├── generate.ts # passthrough to `bun -F @autumn/shared db:generate` -│ ├── migrate.ts # passthrough to `bun -F @autumn/shared db:migrate` +│ ├── migrate.ts # applies pending migrations (CONCURRENTLY-aware executor) │ ├── markApplied.ts # seeds drizzle.__drizzle_migrations │ └── rebase.ts # auto-resolves duplicate-idx conflicts ├── helpers/ +│ ├── applyMigrations.ts # per-migration executor: autocommit for CONCURRENTLY, tx otherwise │ ├── env.ts # --env parsing + infisical wrap + DATABASE_URL host extraction +│ ├── pendingMigrations.ts # computes pending set from _journal.json vs tracking table +│ ├── safetyCheck.ts # flags non-CONCURRENTLY index DDL │ ├── paths.ts # canonical paths to shared/drizzle/ and meta/ │ └── spawn.ts # thin child_process.spawn wrapper └── pull.ts # unrelated — customer data pull (legacy) ``` -`shared/package.json` still owns the implementation of `db:generate` and `db:migrate` (which are what the CLI shells out to under the hood). The unified `bun db` interface lives at the repo root. +`shared/package.json` still owns `db:generate` (which `generate` shells out to). `migrate` no longer delegates to drizzle-kit — it reads the committed migrations with drizzle's `readMigrationFiles` and applies them itself so `CONCURRENTLY` works. The unified `bun db` interface lives at the repo root. --- diff --git a/scripts/db/commands/migrate.ts b/scripts/db/commands/migrate.ts index 5b70dff2e..a2e877bb7 100644 --- a/scripts/db/commands/migrate.ts +++ b/scripts/db/commands/migrate.ts @@ -1,7 +1,8 @@ +import { readMigrationFiles } from "drizzle-orm/migrator"; import pg from "pg"; -import { run } from "../helpers/spawn.ts"; -import { REPO_ROOT } from "../helpers/paths.ts"; +import { MIGRATIONS_DIR } from "../helpers/paths.ts"; import { type Env, targetHost, wrapInInfisical } from "../helpers/env.ts"; +import { applyMigration } from "../helpers/applyMigrations.ts"; import { getPendingMigrations, type PendingMigration, @@ -82,10 +83,41 @@ export async function cmdMigrate( process.exit(1); } - const { code } = await run("bun", ["-F", "@autumn/shared", "db:migrate"], { - cwd: REPO_ROOT, - }); - process.exit(code); + await applyPending(databaseUrl, pending); +} + +/** + * Applies pending migrations using drizzle's own readMigrationFiles (so hashes + * match the tracking table drizzle/mark-applied write) but our own executor, + * which — unlike drizzle's migrate() — can run CONCURRENTLY outside a transaction. + */ +async function applyPending( + databaseUrl: string, + pending: PendingMigration[], +): Promise { + const pendingByMillis = new Map(pending.map((m) => [m.when, m.tag])); + const toApply = readMigrationFiles({ migrationsFolder: MIGRATIONS_DIR }) + .filter((m) => pendingByMillis.has(m.folderMillis)) + .sort((a, b) => a.folderMillis - b.folderMillis); + + const client = new pg.Client({ connectionString: databaseUrl }); + await client.connect(); + try { + for (const migration of toApply) { + const tag = pendingByMillis.get(migration.folderMillis) ?? "migration"; + const { transactional } = await applyMigration(client, migration); + console.log(` applied ${tag}${transactional ? "" : " (concurrent)"}`); + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + console.error(`\nmigration failed: ${message}`); + process.exitCode = 1; + return; + } finally { + await client.end(); + } + + console.log(`done — applied ${toApply.length} migration(s)`); } type FlaggedBlocker = { diff --git a/scripts/db/helpers/applyMigrations.ts b/scripts/db/helpers/applyMigrations.ts new file mode 100644 index 000000000..c9da075cd --- /dev/null +++ b/scripts/db/helpers/applyMigrations.ts @@ -0,0 +1,60 @@ +import type { MigrationMeta } from "drizzle-orm/migrator"; +import type pg from "pg"; + +// CONCURRENTLY (e.g. CREATE INDEX CONCURRENTLY) cannot run inside a transaction +// block. drizzle's own migrate() wraps everything in one transaction, so those +// statements are applied here in autocommit instead. +const NON_TRANSACTIONAL = /\bCONCURRENTLY\b/i; + +const TRACKING_TABLE = `"drizzle"."__drizzle_migrations"`; + +async function recordApplied( + client: pg.Client, + migration: MigrationMeta, +): Promise { + await client.query( + `INSERT INTO ${TRACKING_TABLE} ("hash", "created_at") VALUES ($1, $2)`, + [migration.hash, migration.folderMillis], + ); +} + +export type ApplyResult = { transactional: boolean }; + +/** + * Applies one migration's statements. If any statement is non-transactional + * (CONCURRENTLY), the whole migration runs in autocommit; otherwise it's wrapped + * in a single transaction so DDL + tracking row commit atomically — matching + * drizzle's own per-migration semantics. + */ +export async function applyMigration( + client: pg.Client, + migration: MigrationMeta, +): Promise { + const statements = migration.sql + .map((statement) => statement.trim()) + .filter(Boolean); + const nonTransactional = statements.some((statement) => + NON_TRANSACTIONAL.test(statement), + ); + + if (nonTransactional) { + for (const statement of statements) { + await client.query(statement); + } + await recordApplied(client, migration); + return { transactional: false }; + } + + await client.query("BEGIN"); + try { + for (const statement of statements) { + await client.query(statement); + } + await recordApplied(client, migration); + await client.query("COMMIT"); + } catch (err) { + await client.query("ROLLBACK"); + throw err; + } + return { transactional: true }; +}