diff --git a/package.json b/package.json index 3324a8b38..e6941ea61 100644 --- a/package.json +++ b/package.json @@ -92,9 +92,11 @@ "migrate-functions:prod": "infisical run --env=prod --recursive -- bun scripts/migrations/migrate-functions.ts", "validate-schema": "infisical run --env=prod --recursive -- bun scripts/migrations/validate-schema.ts", "validate-shebangs": "infisical run --env=prod --recursive -- bun scripts/migrations/validate-fullsubject-shebangs.ts", - "tinybird:info": "cd server && infisical run --env=dev --recursive -- bunx tinybird info", - "tinybird:deploy:dev:check": "cd server && infisical run --env=dev --recursive -- bunx tinybird deploy --check", - "tinybird:deploy:dev": "cd server && infisical run --env=dev --recursive -- bunx tinybird deploy", + + "tb": "bun scripts/tinybird/index.ts", + "tb:prod": "bun scripts/tinybird/index.ts prod", + "tb:prod-legacy": "bun scripts/tinybird/index.ts prod-legacy", + "trigger:deploy": "bunx trigger.dev deploy", "setupci": "node scripts/setup/setupci.js", "replicate": "bun scripts/db/replicate.ts", diff --git a/scripts/tinybird/index.ts b/scripts/tinybird/index.ts new file mode 100644 index 000000000..8a967e6f5 --- /dev/null +++ b/scripts/tinybird/index.ts @@ -0,0 +1,137 @@ +import path from "node:path"; + +type ProfileName = "dev" | "prod" | "prod-legacy"; +type TinybirdTarget = "new" | "legacy"; + +type Profile = { + infisicalEnv: "dev" | "prod"; + target: TinybirdTarget; +}; + +const PROFILE_ARG_VALUES = new Set(["prod", "prod-legacy"]); + +const profiles: Record = { + dev: { + infisicalEnv: "dev", + target: "new", + }, + prod: { + infisicalEnv: "prod", + target: "new", + }, + "prod-legacy": { + infisicalEnv: "prod", + target: "legacy", + }, +}; + +const commandAliases: Record = { + info: ["info"], + "deploy:check": ["deploy", "--check"], + deploy: ["deploy"], +}; + +const usage = `Usage: + bun tb info + bun tb deploy:check + bun tb deploy + bun tb:prod + bun tb:prod-legacy + +Targets: + bun tb Infisical dev, new Tinybird instance + bun tb:prod Infisical prod, new Tinybird instance + bun tb:prod-legacy Infisical prod, legacy Tinybird instance`; + +const rootDir = path.resolve(import.meta.dir, "../.."); +const serverDir = path.join(rootDir, "server"); + +const run = async ( + cmd: string[], + options?: { cwd?: string; env?: NodeJS.ProcessEnv }, +) => { + const proc = Bun.spawn(cmd, { + cwd: options?.cwd ?? rootDir, + env: options?.env, + stdin: "inherit", + stdout: "inherit", + stderr: "inherit", + }); + + return proc.exited; +}; + +const requireEnv = (name: string) => { + const value = process.env[name]; + if (!value) { + console.error(`${name} is not set`); + process.exit(1); + } + return value; +}; + +const resolveTinybirdArgs = (args: string[]) => { + if (args.length === 0 || args[0] === "--help" || args[0] === "-h") { + console.log(usage); + process.exit(args.length === 0 ? 1 : 0); + } + + return commandAliases[args[0]] ?? args; +}; + +const executeTinybird = async () => { + const target = requireEnv("AUTUMN_TINYBIRD_TARGET") as TinybirdTarget; + const env = { ...process.env }; + + if (target === "new") { + env.TINYBIRD_API_URL = requireEnv("TINYBIRD_US_EAST_API_URL"); + env.TINYBIRD_TOKEN = requireEnv("TINYBIRD_US_EAST_TOKEN"); + } else { + requireEnv("TINYBIRD_API_URL"); + requireEnv("TINYBIRD_TOKEN"); + } + + const args = resolveTinybirdArgs(Bun.argv.slice(2)); + const exitCode = await run(["bunx", "tinybird", ...args], { + cwd: serverDir, + env, + }); + process.exit(exitCode); +}; + +const main = async () => { + if (process.env.AUTUMN_TINYBIRD_BOOTSTRAPPED === "1") { + await executeTinybird(); + return; + } + + const rawArgs = Bun.argv.slice(2); + const profileName = PROFILE_ARG_VALUES.has(rawArgs[0]) + ? (rawArgs.shift() as ProfileName) + : "dev"; + const profile = profiles[profileName]; + const args = resolveTinybirdArgs(rawArgs); + + const exitCode = await run( + [ + "infisical", + "run", + `--env=${profile.infisicalEnv}`, + "--recursive", + "--", + "bun", + "scripts/tinybird/index.ts", + ...args, + ], + { + env: { + ...process.env, + AUTUMN_TINYBIRD_BOOTSTRAPPED: "1", + AUTUMN_TINYBIRD_TARGET: profile.target, + }, + }, + ); + process.exit(exitCode); +}; + +await main(); diff --git a/server/src/external/aws/ecs/onAwsEcs.ts b/server/src/external/aws/ecs/onAwsEcs.ts new file mode 100644 index 000000000..4204523bd --- /dev/null +++ b/server/src/external/aws/ecs/onAwsEcs.ts @@ -0,0 +1,9 @@ +/** + * True iff this process is running inside an AWS ECS / Fargate task. + * `ECS_CONTAINER_METADATA_URI_V4` is auto-injected by the ECS runtime + * and unset everywhere else (local dev, trigger.dev workers, scripts), + * so it's the canonical "am I on AWS?" gate — same one + * `awsTaskIdentity` uses to discover the running service. + */ +export const onAwsEcs = (): boolean => + Boolean(process.env.ECS_CONTAINER_METADATA_URI_V4); diff --git a/server/src/external/redis/getReachableDragonflyUrl.ts b/server/src/external/redis/getReachableDragonflyUrl.ts new file mode 100644 index 000000000..588426f32 --- /dev/null +++ b/server/src/external/redis/getReachableDragonflyUrl.ts @@ -0,0 +1,30 @@ +import { onAwsEcs } from "@/external/aws/ecs/onAwsEcs.js"; + +/** + * Map a stored dragonfly URL to one that is actually reachable from the + * current process. + * + * Background: orgs whose `redis_config` points at the SHARED dragonfly + * instance store its private VPC URL (`CACHE_V2_DRAGONFLY_URL`) in the + * config row. That URL is only reachable from inside our AWS Fargate + * tasks. Off-AWS callers (local dev, trigger.dev workers) need the + * public mirror `CACHE_V2_DRAGONFLY_PUBLIC_URL` instead. + * + * Heuristic: if the URL equals the private shared URL AND we're not on + * AWS, swap to public. Anything else (per-org dragonfly, Upstash, Redis + * Cloud, etc.) is returned untouched — we have no opinion about those. + * + * Returns the same input string when no swap applies, so callers can use + * it transparently in place of the raw URL. + */ +export const getReachableDragonflyUrl = (url: string): string => { + if (onAwsEcs()) return url; + + const privateUrl = process.env.CACHE_V2_DRAGONFLY_URL?.trim(); + if (!privateUrl || url.trim() !== privateUrl) return url; + + const publicUrl = process.env.CACHE_V2_DRAGONFLY_PUBLIC_URL?.trim(); + if (!publicUrl) return url; + + return publicUrl; +}; diff --git a/server/src/external/redis/initRedisV2.ts b/server/src/external/redis/initRedisV2.ts index 49fd6f09b..f67501e54 100644 --- a/server/src/external/redis/initRedisV2.ts +++ b/server/src/external/redis/initRedisV2.ts @@ -1,6 +1,7 @@ import type { Redis } from "ioredis"; import { logger } from "@/external/logtail/logtailUtils.js"; import type { RedisV2InstanceName } from "@/internal/misc/redisV2Cache/redisV2CacheSchemas.js"; +import { getReachableDragonflyUrl } from "./getReachableDragonflyUrl.js"; import { createRedisConnection, currentRegion, @@ -12,8 +13,13 @@ import { supportsUpstashShebangForRedisV2, } from "./initUtils/redisV2Config.js"; +const rawDragonflyUrl = process.env.CACHE_V2_DRAGONFLY_URL?.trim(); +const dragonflyUrl = rawDragonflyUrl + ? getReachableDragonflyUrl(rawDragonflyUrl) + : undefined; + export const redisV2: Redis = createRedisConnection({ - cacheUrl: process.env.CACHE_V2_DRAGONFLY_URL?.trim() || "", + cacheUrl: dragonflyUrl || "", region: `${currentRegion}:v2`, supportsUpstashShebang: false, commandTimeout: REDIS_V2_COMMAND_TIMEOUT_MS, @@ -22,7 +28,7 @@ export const redisV2: Redis = createRedisConnection({ const alternateInstanceUrls: Partial> = { upstash: process.env.CACHE_V2_UPSTASH_URL?.trim() || undefined, redis: process.env.CACHE_V2_REDIS_URL?.trim() || undefined, - dragonfly: process.env.CACHE_V2_DRAGONFLY_URL?.trim() || undefined, + dragonfly: dragonflyUrl, }; const instancePool = new Map(); diff --git a/server/src/external/redis/orgRedisPool.ts b/server/src/external/redis/orgRedisPool.ts index 8ea2fbf82..b1bfbbadd 100644 --- a/server/src/external/redis/orgRedisPool.ts +++ b/server/src/external/redis/orgRedisPool.ts @@ -4,6 +4,7 @@ import type { DrizzleCli } from "@/db/initDrizzle.js"; import { logger } from "@/external/logtail/logtailUtils.js"; import { OrgService } from "@/internal/orgs/OrgService.js"; import { decryptData } from "@/utils/encryptUtils.js"; +import { getReachableDragonflyUrl } from "./getReachableDragonflyUrl.js"; import { createRedisConnection, currentRegion } from "./initRedis.js"; import { REDIS_V2_COMMAND_TIMEOUT_MS } from "./initUtils/redisV2Config.js"; import { resolveRedisV2 } from "./resolveRedisV2.js"; @@ -69,7 +70,7 @@ export const getOrgRedis = ({ org }: { org: OrgWithRedisConfig }): Redis => { } const instance = createOrgRedisConnection({ - connectionString, + connectionString: getReachableDragonflyUrl(connectionString), orgId: org.id, }); pool.set(org.id, { instance, url: org.redis_config.url });