Revert "Tinybird reads cutover us east"
This commit is contained in:
@@ -58,7 +58,7 @@ Set these in the process environment before running `bun dev:agent` and they wil
|
|||||||
| PostgreSQL | 5432 | Database: `autumn`, user: `postgres`, password: `postgres` |
|
| PostgreSQL | 5432 | Database: `autumn`, user: `postgres`, password: `postgres` |
|
||||||
| Redis Stack | 6379 | Used for `CACHE_URL` and `CACHE_URL_US_EAST` (RedisJSON required) |
|
| Redis Stack | 6379 | Used for `CACHE_URL` and `CACHE_URL_US_EAST` (RedisJSON required) |
|
||||||
| ElasticMQ | 9324 | Local SQS replacement, queue: `autumn.fifo` |
|
| ElasticMQ | 9324 | Local SQS replacement, queue: `autumn.fifo` |
|
||||||
| ClickHouse | 8123 | Used for `TINYBIRD_US_EAST_CLICKHOUSE_URL` |
|
| ClickHouse | 8123 | Used for `TINYBIRD_CLICKHOUSE_URL` |
|
||||||
| Server | 8080 | Autumn API server |
|
| Server | 8080 | Autumn API server |
|
||||||
| Vite | 3000 | Frontend dev server |
|
| Vite | 3000 | Frontend dev server |
|
||||||
| Checkout | 3001 | Checkout app dev server |
|
| Checkout | 3001 | Checkout app dev server |
|
||||||
|
|||||||
@@ -67,7 +67,7 @@ AWS_ACCESS_KEY_ID=x
|
|||||||
AWS_SECRET_ACCESS_KEY=x
|
AWS_SECRET_ACCESS_KEY=x
|
||||||
|
|
||||||
# ClickHouse (local)
|
# ClickHouse (local)
|
||||||
TINYBIRD_US_EAST_CLICKHOUSE_URL=http://localhost:8123
|
TINYBIRD_CLICKHOUSE_URL=http://localhost:8123
|
||||||
|
|
||||||
# App URLs
|
# App URLs
|
||||||
BETTER_AUTH_URL=http://localhost:${serverPort}
|
BETTER_AUTH_URL=http://localhost:${serverPort}
|
||||||
|
|||||||
@@ -3,8 +3,8 @@ import { type ClickHouseClient, createClient } from "@clickhouse/client";
|
|||||||
// ClickHouse URL is different from API URL
|
// ClickHouse URL is different from API URL
|
||||||
// API: https://api.europe-west2.gcp.tinybird.co
|
// API: https://api.europe-west2.gcp.tinybird.co
|
||||||
// ClickHouse: https://europe-west2.gcp.clickhouse.tinybird.co
|
// ClickHouse: https://europe-west2.gcp.clickhouse.tinybird.co
|
||||||
const TINYBIRD_CLICKHOUSE_URL = process.env.TINYBIRD_US_EAST_CLICKHOUSE_URL;
|
const TINYBIRD_CLICKHOUSE_URL = process.env.TINYBIRD_CLICKHOUSE_URL;
|
||||||
const TINYBIRD_TOKEN = process.env.TINYBIRD_US_EAST_TOKEN;
|
const TINYBIRD_TOKEN = process.env.TINYBIRD_TOKEN;
|
||||||
|
|
||||||
// Debug logging
|
// Debug logging
|
||||||
if (TINYBIRD_CLICKHOUSE_URL && TINYBIRD_TOKEN) {
|
if (TINYBIRD_CLICKHOUSE_URL && TINYBIRD_TOKEN) {
|
||||||
|
|||||||
22
server/src/external/tinybird/initTinybirdV2.ts
vendored
22
server/src/external/tinybird/initTinybirdV2.ts
vendored
@@ -1,19 +1,19 @@
|
|||||||
import { createTinybirdApi } from "@tinybirdco/sdk";
|
import { createTinybirdApi } from "@tinybirdco/sdk";
|
||||||
|
|
||||||
// Dual-write safety net during the us-east cutover. Reads the legacy
|
const TINYBIRD_US_EAST_API_URL = process.env.TINYBIRD_US_EAST_API_URL;
|
||||||
// us-west env vars; delete with the dual-write logic in `sendEvents.ts`
|
const TINYBIRD_US_EAST_TOKEN = process.env.TINYBIRD_US_EAST_TOKEN;
|
||||||
// once us-east is stable.
|
|
||||||
const TINYBIRD_API_URL = process.env.TINYBIRD_API_URL;
|
|
||||||
const TINYBIRD_TOKEN = process.env.TINYBIRD_TOKEN;
|
|
||||||
|
|
||||||
/** Secondary Tinybird API client for dual-write during region cutover. */
|
/** Secondary Tinybird API client (us-east region) for dual-write during migration. */
|
||||||
export const tinybirdSecondaryApi =
|
export const tinybirdUsEastApi =
|
||||||
TINYBIRD_API_URL && TINYBIRD_TOKEN
|
TINYBIRD_US_EAST_API_URL && TINYBIRD_US_EAST_TOKEN
|
||||||
? createTinybirdApi({ baseUrl: TINYBIRD_API_URL, token: TINYBIRD_TOKEN })
|
? createTinybirdApi({
|
||||||
|
baseUrl: TINYBIRD_US_EAST_API_URL,
|
||||||
|
token: TINYBIRD_US_EAST_TOKEN,
|
||||||
|
})
|
||||||
: null;
|
: null;
|
||||||
|
|
||||||
if (tinybirdSecondaryApi) {
|
if (tinybirdUsEastApi) {
|
||||||
console.log(
|
console.log(
|
||||||
`[Tinybird] secondary dual-write configured with URL: ${TINYBIRD_API_URL}`,
|
`[Tinybird] us-east dual-write configured with URL: ${TINYBIRD_US_EAST_API_URL}`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import type { EventInsert } from "@autumn/shared";
|
|||||||
import * as Sentry from "@sentry/bun";
|
import * as Sentry from "@sentry/bun";
|
||||||
import type { Logger } from "@/external/logtail/logtailUtils.js";
|
import type { Logger } from "@/external/logtail/logtailUtils.js";
|
||||||
import { tinybirdIngest } from "../initTinybird.js";
|
import { tinybirdIngest } from "../initTinybird.js";
|
||||||
import { tinybirdSecondaryApi } from "../initTinybirdV2.js";
|
import { tinybirdUsEastApi } from "../initTinybirdV2.js";
|
||||||
import { isTinybirdConfigured } from "../tinybirdUtils.js";
|
import { isTinybirdConfigured } from "../tinybirdUtils.js";
|
||||||
import { mapToTinybirdEvent } from "./mapEvent.js";
|
import { mapToTinybirdEvent } from "./mapEvent.js";
|
||||||
|
|
||||||
@@ -35,7 +35,7 @@ export const sendEventsToTinybird = async ({
|
|||||||
|
|
||||||
const tinybirdEvents = events.map(mapToTinybirdEvent);
|
const tinybirdEvents = events.map(mapToTinybirdEvent);
|
||||||
|
|
||||||
const reportFailure = (error: unknown, region: "primary" | "secondary") => {
|
const reportFailure = (error: unknown, region: "primary" | "us-east") => {
|
||||||
const errorId = generateErrorId();
|
const errorId = generateErrorId();
|
||||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||||
|
|
||||||
@@ -81,21 +81,24 @@ export const sendEventsToTinybird = async ({
|
|||||||
})
|
})
|
||||||
.catch((error: unknown) => reportFailure(error, "primary"));
|
.catch((error: unknown) => reportFailure(error, "primary"));
|
||||||
|
|
||||||
const secondaryWrite = tinybirdSecondaryApi
|
const usEastWrite = tinybirdUsEastApi
|
||||||
? tinybirdSecondaryApi
|
? tinybirdUsEastApi
|
||||||
.ingestBatch("events", tinybirdEvents)
|
.ingestBatch("events", tinybirdEvents)
|
||||||
.then((result) => {
|
.then((result) => {
|
||||||
logger?.info(`Sent ${events.length} events to Tinybird (secondary)`, {
|
logger?.info(
|
||||||
data: {
|
`Sent ${events.length} events to Tinybird (us-east)`,
|
||||||
region: "secondary",
|
{
|
||||||
eventCount: events.length,
|
data: {
|
||||||
successfulRows: result?.successful_rows,
|
region: "us-east",
|
||||||
quarantinedRows: result?.quarantined_rows,
|
eventCount: events.length,
|
||||||
|
successfulRows: result?.successful_rows,
|
||||||
|
quarantinedRows: result?.quarantined_rows,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
});
|
);
|
||||||
})
|
})
|
||||||
.catch((error: unknown) => reportFailure(error, "secondary"))
|
.catch((error: unknown) => reportFailure(error, "us-east"))
|
||||||
: Promise.resolve();
|
: Promise.resolve();
|
||||||
|
|
||||||
await Promise.all([primaryWrite, secondaryWrite]);
|
await Promise.all([primaryWrite, usEastWrite]);
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
import { ErrCode, RecaseError } from "@autumn/shared";
|
import { ErrCode, RecaseError } from "@autumn/shared";
|
||||||
import { StatusCodes } from "http-status-codes";
|
import { StatusCodes } from "http-status-codes";
|
||||||
|
|
||||||
const TINYBIRD_API_URL = process.env.TINYBIRD_US_EAST_API_URL;
|
const TINYBIRD_API_URL = process.env.TINYBIRD_API_URL;
|
||||||
const TINYBIRD_TOKEN = process.env.TINYBIRD_US_EAST_TOKEN;
|
const TINYBIRD_TOKEN = process.env.TINYBIRD_TOKEN;
|
||||||
|
|
||||||
export type TinybirdConfig = {
|
export type TinybirdConfig = {
|
||||||
baseUrl: string;
|
baseUrl: string;
|
||||||
|
|||||||
Reference in New Issue
Block a user