diff --git a/package.json b/package.json index 1fecf8082..8e5736abb 100644 --- a/package.json +++ b/package.json @@ -74,6 +74,7 @@ "p": "ENV_FILE=.env.prod infisical run --env=prod -- bun scripts/dev.ts", "l": "bash ./scripts/dev-local.sh", "setup": "node scripts/setup/setup.js", + "setup:s3-admin": "bun scripts/setup/setupS3Admin.ts", "setup:test": "infisical run --env=dev -- bun scripts/setup/setup-test.ts", "migrate-functions": "infisical run --env=dev -- bun scripts/migrations/migrate-functions.ts", "migrate-functions:test": "infisical run --env=test -- bun scripts/migrations/migrate-functions.ts", diff --git a/scripts/package.json b/scripts/package.json index 50f5f6a62..e3baba8f7 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -9,6 +9,7 @@ "replicate": "bun run db/replicate.ts" }, "dependencies": { + "@aws-sdk/client-s3": "^3.1017.0", "@autumn/shared": "workspace:*", "chalk": "^5.3.0", "dotenv": "^16.5.0", diff --git a/scripts/setup/setupS3Admin.ts b/scripts/setup/setupS3Admin.ts new file mode 100644 index 000000000..131af2042 --- /dev/null +++ b/scripts/setup/setupS3Admin.ts @@ -0,0 +1,203 @@ +#!/usr/bin/env bun +import { + BucketAlreadyExists, + BucketAlreadyOwnedByYou, + type BucketLocationConstraint, + CreateBucketCommand, + HeadBucketCommand, + HeadObjectCommand, + PutObjectCommand, + S3Client, +} from "@aws-sdk/client-s3"; +import { + ADMIN_REQUEST_BLOCK_CONFIG_KEY as REQUEST_BLOCK_CONFIG_KEY, + getAdminS3Config, +} from "@server/external/aws/s3/adminS3Config.js"; +const DEFAULT_REQUEST_BLOCK_CONFIG = { + orgs: {}, +}; + +const getTargetFromArgs = () => { + const hasDevFlag = process.argv.includes("--dev"); + const hasProdFlag = process.argv.includes("--prod"); + + if (hasDevFlag && hasProdFlag) { + throw new Error("Use either --dev or --prod, not both"); + } + + if (hasDevFlag) return "dev" as const; + if (hasProdFlag) return "prod" as const; + return undefined; +}; + +const createS3Client = ({ region }: { region: string }) => { + return new S3Client({ region }); +}; + +const getHttpStatusCode = ({ error }: { error: unknown }) => { + if (!error || typeof error !== "object") return undefined; + + const errorWithMetadata = error as { + $metadata?: { + httpStatusCode?: number; + }; + }; + + return errorWithMetadata.$metadata?.httpStatusCode; +}; + +const isMissingS3ResourceError = ({ error }: { error: unknown }) => { + if (error instanceof Error) { + if (error.name === "NotFound" || error.name === "NoSuchBucket") { + return true; + } + } + + return getHttpStatusCode({ error }) === 404; +}; + +const bucketExists = async ({ + s3Client, + bucket, +}: { + s3Client: S3Client; + bucket: string; +}) => { + try { + await s3Client.send( + new HeadBucketCommand({ + Bucket: bucket, + }), + ); + return true; + } catch (error) { + if (isMissingS3ResourceError({ error })) { + return false; + } + + throw error; + } +}; + +const ensureBucketExists = async ({ + s3Client, + bucket, + region, +}: { + s3Client: S3Client; + bucket: string; + region: string; +}) => { + const exists = await bucketExists({ s3Client, bucket }); + if (exists) { + console.log(`Bucket already exists: ${bucket}`); + return; + } + + try { + await s3Client.send( + new CreateBucketCommand({ + Bucket: bucket, + ...(region === "us-east-1" + ? {} + : { + CreateBucketConfiguration: { + LocationConstraint: region as BucketLocationConstraint, + }, + }), + }), + ); + console.log(`Created bucket: ${bucket}`); + } catch (error) { + if ( + error instanceof BucketAlreadyOwnedByYou || + error instanceof BucketAlreadyExists + ) { + console.log(`Bucket already exists: ${bucket}`); + return; + } + + throw error; + } +}; + +const objectExists = async ({ + s3Client, + bucket, + key, +}: { + s3Client: S3Client; + bucket: string; + key: string; +}) => { + try { + await s3Client.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); + return true; + } catch (error) { + if (isMissingS3ResourceError({ error })) { + return false; + } + + throw error; + } +}; + +const ensureRequestBlockConfigExists = async ({ + s3Client, + bucket, +}: { + s3Client: S3Client; + bucket: string; +}) => { + const exists = await objectExists({ + s3Client, + bucket, + key: REQUEST_BLOCK_CONFIG_KEY, + }); + + if (exists) { + console.log(`Admin config already exists: ${REQUEST_BLOCK_CONFIG_KEY}`); + return; + } + + await s3Client.send( + new PutObjectCommand({ + Bucket: bucket, + Key: REQUEST_BLOCK_CONFIG_KEY, + Body: JSON.stringify(DEFAULT_REQUEST_BLOCK_CONFIG, null, 2), + ContentType: "application/json", + }), + ); + + console.log(`Created admin config: ${REQUEST_BLOCK_CONFIG_KEY}`); +}; + +const main = async () => { + const target = getTargetFromArgs(); + const { bucket, region } = getAdminS3Config({ target }); + const s3Client = createS3Client({ region }); + + console.log( + `Initializing S3 admin config for ${target || process.env.NODE_ENV || "default"} -> s3://${bucket}/${REQUEST_BLOCK_CONFIG_KEY}`, + ); + + await ensureBucketExists({ + s3Client, + bucket, + region, + }); + + await ensureRequestBlockConfigExists({ + s3Client, + bucket, + }); + + console.log("S3 admin initialization complete."); +}; + +await main(); diff --git a/server/src/external/aws/s3/adminS3Config.ts b/server/src/external/aws/s3/adminS3Config.ts new file mode 100644 index 000000000..b6224a47d --- /dev/null +++ b/server/src/external/aws/s3/adminS3Config.ts @@ -0,0 +1,29 @@ +export const ADMIN_REQUEST_BLOCK_CONFIG_KEY = "admin/request-block-config.json"; + +type AdminS3Target = "dev" | "prod"; + +const isDevTarget = ({ target }: { target?: AdminS3Target }) => { + if (target) return target === "dev"; + return ( + process.env.NODE_ENV === "dev" || + process.env.NODE_ENV === "development" + ); +}; + +export const getAdminS3Config = ({ + target, +}: { + target?: AdminS3Target; +} = {}) => { + if (isDevTarget({ target })) { + return { + bucket: "autumn-dev-server", + region: "eu-west-2", + }; + } + + return { + bucket: "autumn-prod-server", + region: "us-east-2", + }; +}; diff --git a/server/src/external/aws/s3/initS3.ts b/server/src/external/aws/s3/initS3.ts new file mode 100644 index 000000000..251b36a8d --- /dev/null +++ b/server/src/external/aws/s3/initS3.ts @@ -0,0 +1,17 @@ +import { S3Client } from "@aws-sdk/client-s3"; +import { DEFAULT_AWS_REGION } from "@/external/aws/awsRegionUtils.js"; + +const s3ClientsByRegion = new Map(); + +export const getS3Client = ({ + region = DEFAULT_AWS_REGION, +}: { + region?: string; +}) => { + const existingClient = s3ClientsByRegion.get(region); + if (existingClient) return existingClient; + + const s3Client = new S3Client({ region }); + s3ClientsByRegion.set(region, s3Client); + return s3Client; +}; diff --git a/server/src/external/aws/s3/s3Utils.ts b/server/src/external/aws/s3/s3Utils.ts new file mode 100644 index 000000000..f9494765b --- /dev/null +++ b/server/src/external/aws/s3/s3Utils.ts @@ -0,0 +1,11 @@ +export const getS3BodyAsString = async ({ + body, +}: { + body: { transformToString?: () => Promise }; +}) => { + if (typeof body.transformToString === "function") { + return await body.transformToString(); + } + + return await new Response(body as BodyInit).text(); +}; diff --git a/server/src/honoMiddlewares/requestBlockMiddleware.ts b/server/src/honoMiddlewares/requestBlockMiddleware.ts index 8a2b0c80a..09af1288d 100644 --- a/server/src/honoMiddlewares/requestBlockMiddleware.ts +++ b/server/src/honoMiddlewares/requestBlockMiddleware.ts @@ -1,8 +1,8 @@ import { ErrCode, RecaseError } from "@autumn/shared"; import type { Context, Next } from "hono"; import type { HonoEnv } from "@/honoUtils/HonoEnv.js"; +import { getRuntimeRequestBlockEntry } from "@/internal/misc/requestBlocks/requestBlockStore.js"; import { matchRoute } from "./middlewareUtils.js"; -import { getRuntimeRequestBlockEntry } from "@/internal/requestBlocks/requestBlockStore.js"; export const requestBlockMiddleware = async ( c: Context, @@ -22,14 +22,10 @@ export const requestBlockMiddleware = async ( return; } + console.log("Entry:", entry); + if (entry.blockAll) { - ctx.logger.warn("Rejecting blocked /v1 request", { - orgId, - orgSlug: ctx.org?.slug, - method: c.req.method, - path: c.req.path, - blockAll: true, - }); + ctx.logger.warn("Rejecting blocked /v1 request (block all)"); throw new RecaseError({ message: "API access is temporarily disabled for this organization", @@ -54,13 +50,9 @@ export const requestBlockMiddleware = async ( return; } - ctx.logger.warn("Rejecting endpoint-blocked /v1 request", { - orgId, - orgSlug: ctx.org?.slug, - method: c.req.method, - path: c.req.path, - rule: matchedRule, - }); + ctx.logger.warn( + "Rejecting endpoint-blocked /v1 request (blocked endpoint, matched rule)", + ); throw new RecaseError({ message: "This endpoint is temporarily disabled for this organization", diff --git a/server/src/init.ts b/server/src/init.ts index 05de97bd9..edd9cd9d7 100644 --- a/server/src/init.ts +++ b/server/src/init.ts @@ -14,7 +14,7 @@ import { logger } from "./external/logtail/logtailUtils.js"; import { startRequestBlockPolling, stopRequestBlockPolling, -} from "./internal/requestBlocks/requestBlockStore.js"; +} from "./internal/misc/requestBlocks/requestBlockStore.js"; import { warmupRegionalRedis } from "./external/redis/initRedis.js"; import { createHonoApp } from "./initHono.js"; import { otelSdk } from "./instrumentation.js"; diff --git a/server/src/internal/admin/handleGetAdminOrgRequestBlock.ts b/server/src/internal/admin/handleGetAdminOrgRequestBlock.ts index 859e725cf..fd272c740 100644 --- a/server/src/internal/admin/handleGetAdminOrgRequestBlock.ts +++ b/server/src/internal/admin/handleGetAdminOrgRequestBlock.ts @@ -3,7 +3,7 @@ import { createRoute } from "@/honoMiddlewares/routeHandler.js"; import { getOrgRequestBlockFromSource, getRuntimeRequestBlockStatus, -} from "@/internal/requestBlocks/requestBlockStore.js"; +} from "@/internal/misc/requestBlocks/requestBlockStore.js"; export const handleGetAdminOrgRequestBlock = createRoute({ params: z.object({ diff --git a/server/src/internal/admin/handleListAdminOrgs.ts b/server/src/internal/admin/handleListAdminOrgs.ts index 4b7337992..9dc47936c 100644 --- a/server/src/internal/admin/handleListAdminOrgs.ts +++ b/server/src/internal/admin/handleListAdminOrgs.ts @@ -1,6 +1,6 @@ import { member, organizations, user } from "@autumn/shared"; import { and, desc, eq, gt, gte, ilike, inArray, lt, or } from "drizzle-orm"; -import { getRequestBlockConfigFromSource } from "../requestBlocks/requestBlockStore.js"; +import { getRequestBlockConfigFromSource } from "../misc/requestBlocks/requestBlockStore.js"; import { createRoute } from "../../honoMiddlewares/routeHandler"; export const handleListAdminOrgs = createRoute({ diff --git a/server/src/internal/admin/handleUpsertAdminOrgRequestBlock.ts b/server/src/internal/admin/handleUpsertAdminOrgRequestBlock.ts index 1bf18602a..f66164aeb 100644 --- a/server/src/internal/admin/handleUpsertAdminOrgRequestBlock.ts +++ b/server/src/internal/admin/handleUpsertAdminOrgRequestBlock.ts @@ -1,7 +1,7 @@ import { z } from "zod/v4"; import { createRoute } from "@/honoMiddlewares/routeHandler.js"; -import { updateOrgRequestBlockInSource } from "@/internal/requestBlocks/requestBlockStore.js"; -import { RequestBlockUpdateSchema } from "@/internal/requestBlocks/requestBlockSchemas.js"; +import { updateOrgRequestBlockInSource } from "@/internal/misc/requestBlocks/requestBlockStore.js"; +import { RequestBlockUpdateSchema } from "@/internal/misc/requestBlocks/requestBlockSchemas.js"; export const handleUpsertAdminOrgRequestBlock = createRoute({ params: z.object({ diff --git a/server/src/internal/debug/debugRouter.ts b/server/src/internal/misc/debug/debugRouter.ts similarity index 100% rename from server/src/internal/debug/debugRouter.ts rename to server/src/internal/misc/debug/debugRouter.ts diff --git a/server/src/internal/requestBlocks/requestBlockSchemas.ts b/server/src/internal/misc/requestBlocks/requestBlockSchemas.ts similarity index 100% rename from server/src/internal/requestBlocks/requestBlockSchemas.ts rename to server/src/internal/misc/requestBlocks/requestBlockSchemas.ts diff --git a/server/src/internal/requestBlocks/requestBlockStore.ts b/server/src/internal/misc/requestBlocks/requestBlockStore.ts similarity index 80% rename from server/src/internal/requestBlocks/requestBlockStore.ts rename to server/src/internal/misc/requestBlocks/requestBlockStore.ts index 954bf64aa..6c10683d3 100644 --- a/server/src/internal/requestBlocks/requestBlockStore.ts +++ b/server/src/internal/misc/requestBlocks/requestBlockStore.ts @@ -1,11 +1,16 @@ -import { GetObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3"; +import { ErrCode } from "@autumn/shared"; +import { GetObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3"; +import { + ADMIN_REQUEST_BLOCK_CONFIG_KEY, + getAdminS3Config, +} from "@/external/aws/s3/adminS3Config.js"; +import { getS3Client } from "@/external/aws/s3/initS3.js"; +import { getS3BodyAsString } from "@/external/aws/s3/s3Utils.js"; import type { Logger } from "@/external/logtail/logtailUtils.js"; import RecaseError from "@/utils/errorUtils.js"; -import { ErrCode } from "@autumn/shared"; -import { DEFAULT_AWS_REGION } from "@/external/aws/awsRegionUtils.js"; import { - RequestBlockConfigSchema, type RequestBlockConfig, + RequestBlockConfigSchema, type RequestBlockEntry, type RequestBlockUpdate, } from "./requestBlockSchemas.js"; @@ -31,10 +36,8 @@ let runtimeStatus: RequestBlockStatus = { let pollTimer: ReturnType | null = null; const getConfigLocation = () => { - const bucket = process.env.REQUEST_BLOCK_CONFIG_S3_BUCKET; - const key = process.env.REQUEST_BLOCK_CONFIG_S3_KEY; - const region = - process.env.REQUEST_BLOCK_CONFIG_S3_REGION || DEFAULT_AWS_REGION; + const { bucket, region } = getAdminS3Config(); + const key = ADMIN_REQUEST_BLOCK_CONFIG_KEY; return { bucket, @@ -44,27 +47,14 @@ const getConfigLocation = () => { }; }; -const getS3Client = () => { - const { region } = getConfigLocation(); - return new S3Client({ region }); -}; - -const streamToString = async (body: { transformToString?: () => Promise }) => { - if (typeof body.transformToString === "function") { - return await body.transformToString(); - } - - return await new Response(body as BodyInit).text(); -}; - const readConfigFromS3 = async (): Promise => { - const { bucket, key, configured } = getConfigLocation(); + const { bucket, key, configured, region } = getConfigLocation(); if (!configured || !bucket || !key) { return emptyConfig(); } - const client = getS3Client(); + const client = getS3Client({ region }); try { const response = await client.send( new GetObjectCommand({ @@ -77,7 +67,7 @@ const readConfigFromS3 = async (): Promise => { return emptyConfig(); } - const raw = (await streamToString(response.Body)).trim(); + const raw = (await getS3BodyAsString({ body: response.Body })).trim(); if (!raw) { return emptyConfig(); } @@ -93,7 +83,7 @@ const readConfigFromS3 = async (): Promise => { }; const writeConfigToS3 = async (config: RequestBlockConfig) => { - const { bucket, key, configured } = getConfigLocation(); + const { bucket, key, configured, region } = getConfigLocation(); if (!configured || !bucket || !key) { throw new RecaseError({ @@ -103,7 +93,7 @@ const writeConfigToS3 = async (config: RequestBlockConfig) => { }); } - const client = getS3Client(); + const client = getS3Client({ region }); await client.send( new PutObjectCommand({ Bucket: bucket, @@ -166,7 +156,7 @@ export const refreshRequestBlockConfig = async ({ lastSuccessAt: runtimeStatus.lastSuccessAt, error: error instanceof Error ? error.message : "Failed to load config", }; - logger?.error("Failed to refresh request block config", { error }); + logger?.warn(`Failed to refresh request block config: ${error}`); } }; @@ -211,8 +201,7 @@ export const updateOrgRequestBlockInSource = async ({ updatedBy?: string; }) => { const config = await readConfigFromS3(); - const shouldDelete = - !update.blockAll && update.blockedEndpoints.length === 0; + const shouldDelete = !update.blockAll && update.blockedEndpoints.length === 0; if (shouldDelete) { delete config.orgs[orgId];