comi-logic/cfw-autumn
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "Tinybird reads cutover us east"

Browse Source
This commit is contained in:
Owen
2026-05-18 19:42:14 +01:00
committed by GitHub
parent 1837632cc8
commit 78f35b5273
6 changed files with 33 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scripts/setup/AGENT_README.md
View File

@@ -58,7 +58,7 @@ Set these in the process environment before running `bun dev:agent` and they wil
| PostgreSQL | 5432 | Database: `autumn`, user: `postgres`, password: `postgres` |
| Redis Stack | 6379 | Used for `CACHE_URL` and `CACHE_URL_US_EAST` (RedisJSON required) |
| ElasticMQ | 9324 | Local SQS replacement, queue: `autumn.fifo` |
| ClickHouse | 8123 | Used for `TINYBIRD_US_EAST_CLICKHOUSE_URL` |
| ClickHouse | 8123 | Used for `TINYBIRD_CLICKHOUSE_URL` |
| Server | 8080 | Autumn API server |
| Vite | 3000 | Frontend dev server |
| Checkout | 3001 | Checkout app dev server |

2
scripts/setup/writeAgentEnv.ts
View File

@@ -67,7 +67,7 @@ AWS_ACCESS_KEY_ID=x
AWS_SECRET_ACCESS_KEY=x
# ClickHouse (local)
TINYBIRD_US_EAST_CLICKHOUSE_URL=http://localhost:8123
TINYBIRD_CLICKHOUSE_URL=http://localhost:8123
# App URLs
BETTER_AUTH_URL=http://localhost:${serverPort}

4
server/src/external/tinybird/initClickhouse.ts vendored
View File

@@ -3,8 +3,8 @@ import { type ClickHouseClient, createClient } from "@clickhouse/client";
// ClickHouse URL is different from API URL
// API: https://api.europe-west2.gcp.tinybird.co
// ClickHouse: https://europe-west2.gcp.clickhouse.tinybird.co
const TINYBIRD_CLICKHOUSE_URL = process.env.TINYBIRD_US_EAST_CLICKHOUSE_URL;
const TINYBIRD_TOKEN = process.env.TINYBIRD_US_EAST_TOKEN;
const TINYBIRD_CLICKHOUSE_URL = process.env.TINYBIRD_CLICKHOUSE_URL;
const TINYBIRD_TOKEN = process.env.TINYBIRD_TOKEN;
// Debug logging
if (TINYBIRD_CLICKHOUSE_URL && TINYBIRD_TOKEN) {

22
server/src/external/tinybird/initTinybirdV2.ts vendored
View File

@@ -1,19 +1,19 @@
import { createTinybirdApi } from "@tinybirdco/sdk";
// Dual-write safety net during the us-east cutover. Reads the legacy
// us-west env vars; delete with the dual-write logic in `sendEvents.ts`
// once us-east is stable.
const TINYBIRD_API_URL = process.env.TINYBIRD_API_URL;
const TINYBIRD_TOKEN = process.env.TINYBIRD_TOKEN;
const TINYBIRD_US_EAST_API_URL = process.env.TINYBIRD_US_EAST_API_URL;
const TINYBIRD_US_EAST_TOKEN = process.env.TINYBIRD_US_EAST_TOKEN;
/** Secondary Tinybird API client for dual-write during region cutover. */
export const tinybirdSecondaryApi =
TINYBIRD_API_URL && TINYBIRD_TOKEN
? createTinybirdApi({ baseUrl: TINYBIRD_API_URL, token: TINYBIRD_TOKEN })
/** Secondary Tinybird API client (us-east region) for dual-write during migration. */
export const tinybirdUsEastApi =
TINYBIRD_US_EAST_API_URL && TINYBIRD_US_EAST_TOKEN
? createTinybirdApi({
baseUrl: TINYBIRD_US_EAST_API_URL,
token: TINYBIRD_US_EAST_TOKEN,
})
: null;
if (tinybirdSecondaryApi) {
if (tinybirdUsEastApi) {
console.log(
`[Tinybird] secondary dual-write configured with URL: ${TINYBIRD_API_URL}`,
`[Tinybird] us-east dual-write configured with URL: ${TINYBIRD_US_EAST_API_URL}`,
);
}

29
server/src/external/tinybird/sendEvents/sendEvents.ts vendored
View File

@@ -3,7 +3,7 @@ import type { EventInsert } from "@autumn/shared";
import * as Sentry from "@sentry/bun";
import type { Logger } from "@/external/logtail/logtailUtils.js";
import { tinybirdIngest } from "../initTinybird.js";
import { tinybirdSecondaryApi } from "../initTinybirdV2.js";
import { tinybirdUsEastApi } from "../initTinybirdV2.js";
import { isTinybirdConfigured } from "../tinybirdUtils.js";
import { mapToTinybirdEvent } from "./mapEvent.js";
@@ -35,7 +35,7 @@ export const sendEventsToTinybird = async ({
const tinybirdEvents = events.map(mapToTinybirdEvent);
const reportFailure = (error: unknown, region: "primary" | "secondary") => {
const reportFailure = (error: unknown, region: "primary" | "us-east") => {
const errorId = generateErrorId();
const errorMessage = error instanceof Error ? error.message : String(error);
@@ -81,21 +81,24 @@ export const sendEventsToTinybird = async ({
})
.catch((error: unknown) => reportFailure(error, "primary"));
const secondaryWrite = tinybirdSecondaryApi
? tinybirdSecondaryApi
const usEastWrite = tinybirdUsEastApi
? tinybirdUsEastApi
.ingestBatch("events", tinybirdEvents)
.then((result) => {
logger?.info(`Sent ${events.length} events to Tinybird (secondary)`, {
data: {
region: "secondary",
eventCount: events.length,
successfulRows: result?.successful_rows,
quarantinedRows: result?.quarantined_rows,
logger?.info(
`Sent ${events.length} events to Tinybird (us-east)`,
{
data: {
region: "us-east",
eventCount: events.length,
successfulRows: result?.successful_rows,
quarantinedRows: result?.quarantined_rows,
},
},
});
);
})
.catch((error: unknown) => reportFailure(error, "secondary"))
.catch((error: unknown) => reportFailure(error, "us-east"))
: Promise.resolve();
await Promise.all([primaryWrite, secondaryWrite]);
await Promise.all([primaryWrite, usEastWrite]);
};

4
server/src/external/tinybird/tinybirdUtils.ts vendored
View File

@@ -1,8 +1,8 @@
import { ErrCode, RecaseError } from "@autumn/shared";
import { StatusCodes } from "http-status-codes";
const TINYBIRD_API_URL = process.env.TINYBIRD_US_EAST_API_URL;
const TINYBIRD_TOKEN = process.env.TINYBIRD_US_EAST_TOKEN;
const TINYBIRD_API_URL = process.env.TINYBIRD_API_URL;
const TINYBIRD_TOKEN = process.env.TINYBIRD_TOKEN;
export type TinybirdConfig = {
baseUrl: string;