diff --git a/server/src/external/tinybird/initTinybird.ts b/server/src/external/tinybird/initTinybird.ts index f5fe42a28..e4fbe708f 100644 --- a/server/src/external/tinybird/initTinybird.ts +++ b/server/src/external/tinybird/initTinybird.ts @@ -5,7 +5,6 @@ import { createAggregatePipe } from "./pipes/aggregatePipe.js"; import { createAggregateSimplePipe } from "./pipes/aggregateSimplePipe.js"; import { createListEventNamesPipe } from "./pipes/listEventNamesPipe.js"; import { createListEventsPaginatedPipe } from "./pipes/listEventsPaginatedPipe.js"; -import { createListEventsPipe } from "./pipes/listEventsPipe.js"; const TINYBIRD_API_URL = process.env.TINYBIRD_API_URL; const TINYBIRD_TOKEN = process.env.TINYBIRD_TOKEN; @@ -48,7 +47,6 @@ export const tinybirdPipes = tinybirdClient aggregate: createAggregatePipe(tinybirdClient), aggregateSimple: createAggregateSimplePipe(tinybirdClient), aggregateGroupable: createAggregateGroupablePipe(tinybirdClient), - listEvents: createListEventsPipe(tinybirdClient), listEventNames: createListEventNamesPipe(tinybirdClient), listEventsPaginated: createListEventsPaginatedPipe(tinybirdClient), } @@ -98,6 +96,4 @@ export type { ListEventNamesPipeRow, ListEventsPaginatedPipeParams, ListEventsPaginatedPipeRow, - ListEventsPipeParams, - ListEventsPipeRow, } from "./pipes/index.js"; diff --git a/server/src/external/tinybird/pipes/index.ts b/server/src/external/tinybird/pipes/index.ts index 4b55c7fc1..30af48b57 100644 --- a/server/src/external/tinybird/pipes/index.ts +++ b/server/src/external/tinybird/pipes/index.ts @@ -33,10 +33,3 @@ export { listEventsPaginatedPipeParamsSchema, listEventsPaginatedPipeResponseSchema, } from "./listEventsPaginatedPipe.js"; -export { - createListEventsPipe, - type ListEventsPipeParams, - type ListEventsPipeRow, - listEventsPipeParamsSchema, - listEventsPipeResponseSchema, -} from "./listEventsPipe.js"; diff --git a/server/src/external/tinybird/pipes/listEventsPipe.ts b/server/src/external/tinybird/pipes/listEventsPipe.ts deleted file mode 100644 index 3b3277870..000000000 --- a/server/src/external/tinybird/pipes/listEventsPipe.ts +++ /dev/null @@ -1,41 +0,0 @@ -import type { Tinybird } from "@chronark/zod-bird"; -import { z } from "zod"; - -/** Response schema for the list_events pipe */ -export const listEventsPipeResponseSchema = z.object({ - id: z.string(), - org_id: z.string(), - env: z.string(), - customer_id: z.string(), - event_name: z.string(), - timestamp: z.string(), - value: z.number().nullable(), - properties: z.string().nullable(), - idempotency_key: z.string().nullable(), - entity_id: z.string().nullable(), -}); - -export type ListEventsPipeRow = z.infer; - -/** Parameters schema for the list_events pipe */ -export const listEventsPipeParamsSchema = z.object({ - org_id: z.string(), - env: z.string(), - start_date: z.string(), - end_date: z.string(), - customer_id: z.string().optional(), - event_name: z.string().optional(), - limit: z.number().optional(), - cursor_timestamp: z.string().optional(), - cursor_id: z.string().optional(), -}); - -export type ListEventsPipeParams = z.infer; - -/** Creates the list_events pipe caller */ -export const createListEventsPipe = (tb: Tinybird) => - tb.buildPipe({ - pipe: "list_events", - parameters: listEventsPipeParamsSchema, - data: listEventsPipeResponseSchema, - }); diff --git a/server/src/internal/analytics/actions/aggregate.ts b/server/src/internal/analytics/actions/aggregate.ts index a08f84d3e..a198f56db 100644 --- a/server/src/internal/analytics/actions/aggregate.ts +++ b/server/src/internal/analytics/actions/aggregate.ts @@ -3,20 +3,18 @@ import { type BillingCycleIntervalEnum, type BillingCycleResult, type ClickHouseResult, - ErrCode, - RecaseError, type TimeseriesEventsParams, } from "@autumn/shared"; import { UTCDate } from "@date-fns/utc"; import { addDays, addHours, addMonths, format, sub } from "date-fns"; import { Decimal } from "decimal.js"; -import { StatusCodes } from "http-status-codes"; import { type AggregateGroupablePipeRow, type AggregateSimplePipeRow, getTinybirdPipes, } from "@/external/tinybird/initTinybird.js"; import type { AutumnContext } from "@/honoUtils/HonoEnv.js"; +import { validatePropertyPathForJSON } from "@/internal/analytics/actions/eventValidationUtils.js"; import { getBillingCycleStartDate } from "../analyticsUtils.js"; const DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; @@ -359,20 +357,7 @@ export const aggregate = async ({ propertyKey = params.group_by; } - // Validate property path segments (matches old ClickHouse behavior) - if (groupColumn === "property" && propertyKey) { - const pathSegments = propertyKey.split("."); - for (const segment of pathSegments) { - if (!/^[a-zA-Z0-9_]+$/.test(segment)) { - throw new RecaseError({ - message: - "Invalid property path. Should only contain alphanumeric and underscore characters.", - code: ErrCode.InvalidInputs, - statusCode: StatusCodes.BAD_REQUEST, - }); - } - } - } + validatePropertyPathForJSON({ propertyKey }); const pipeParams = { org_id: org.id, diff --git a/server/src/internal/analytics/actions/eventActions.ts b/server/src/internal/analytics/actions/eventActions.ts index 05a533b41..5fc826613 100644 --- a/server/src/internal/analytics/actions/eventActions.ts +++ b/server/src/internal/analytics/actions/eventActions.ts @@ -1,9 +1,9 @@ -import { aggregate } from "./aggregate.js"; +import { aggregate } from "./eventValidationUtils.js"; import { getCountAndSum } from "./getCountAndSum.js"; import { getEventById } from "./getEventById.js"; import { getTopEventNames } from "./getTopEventNames.js"; import { listEventNames } from "./listEventNames.js"; -import { listEventsForApi } from "./listEventsForApi.js"; +import { listEvents } from "./listEvents.js"; import { listRawEvents } from "./listRawEvents.js"; export const eventActions = { @@ -12,6 +12,6 @@ export const eventActions = { getEventById, getTopEventNames, listEventNames, - listEventsForApi, + listEvents, listRawEvents, } as const; diff --git a/server/src/internal/analytics/actions/eventValidationUtils.ts b/server/src/internal/analytics/actions/eventValidationUtils.ts new file mode 100644 index 000000000..959861885 --- /dev/null +++ b/server/src/internal/analytics/actions/eventValidationUtils.ts @@ -0,0 +1,21 @@ +import { ErrCode, RecaseError } from "@shared/index"; +import { StatusCodes } from "http-status-codes"; + +export const validatePropertyPathForJSON = ({ + propertyKey, +}: { + propertyKey: string; +}) => { + // Validate property path segments (matches old ClickHouse behavior) + const pathSegments = propertyKey.split("."); + for (const segment of pathSegments) { + if (!/^[a-zA-Z0-9_]+$/.test(segment)) { + throw new RecaseError({ + message: + "Invalid property path. Should only contain alphanumeric and underscore characters.", + code: ErrCode.InvalidInputs, + statusCode: StatusCodes.BAD_REQUEST, + }); + } + } +}; diff --git a/server/src/internal/analytics/actions/listEventsForApi.ts b/server/src/internal/analytics/actions/listEvents.ts similarity index 78% rename from server/src/internal/analytics/actions/listEventsForApi.ts rename to server/src/internal/analytics/actions/listEvents.ts index f78135a9c..4aef9abba 100644 --- a/server/src/internal/analytics/actions/listEventsForApi.ts +++ b/server/src/internal/analytics/actions/listEvents.ts @@ -1,21 +1,10 @@ import type { ApiEventsListItem } from "@autumn/shared"; +import { epochToDateTime } from "@autumn/shared/api/common/epochUtils"; import { getTinybirdPipes } from "@/external/tinybird/initTinybird.js"; import type { AutumnContext } from "@/honoUtils/HonoEnv.js"; -/** Converts epoch ms to ClickHouse DateTime string format */ -const epochToDateTime = (epochMs: number): string => { - const date = new Date(epochMs); - const year = date.getUTCFullYear(); - const month = String(date.getUTCMonth() + 1).padStart(2, "0"); - const day = String(date.getUTCDate()).padStart(2, "0"); - const hours = String(date.getUTCHours()).padStart(2, "0"); - const minutes = String(date.getUTCMinutes()).padStart(2, "0"); - const seconds = String(date.getUTCSeconds()).padStart(2, "0"); - return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`; -}; - /** Lists events for the external API with offset-based pagination */ -export const listEventsForApi = async ({ +export const listEvents = async ({ ctx, params, }: { diff --git a/server/src/internal/analytics/actions/listRawEvents.ts b/server/src/internal/analytics/actions/listRawEvents.ts index 099d673fb..42ea2ddde 100644 --- a/server/src/internal/analytics/actions/listRawEvents.ts +++ b/server/src/internal/analytics/actions/listRawEvents.ts @@ -6,7 +6,7 @@ import type { } from "@autumn/shared"; import { getTinybirdPipes, - type ListEventsPipeRow, + type ListEventsPaginatedPipeRow, } from "@/external/tinybird/initTinybird.js"; import type { AutumnContext } from "@/honoUtils/HonoEnv.js"; import { getBillingCycleStartDate } from "../analyticsUtils.js"; @@ -50,7 +50,7 @@ const calculateStartDateFromInterval = (interval: string): Date => { /** Converts pipe row to the expected ClickHouse format */ const convertPipeRowToClickHouseFormat = ( - row: ListEventsPipeRow, + row: ListEventsPaginatedPipeRow, ): RawEventFromClickHouse => ({ id: row.id, customer_id: row.customer_id, @@ -67,8 +67,6 @@ export type ListRawEventsParams = { aggregateAll?: boolean; event_name?: string; limit?: number; - cursor_timestamp?: string; - cursor_id?: string; }; /** Lists raw events with optional filtering by customer and date range */ @@ -114,10 +112,9 @@ export const listRawEvents = async ({ start_date: finalStartDate, end_date: finalEndDate, customer_id: params.aggregateAll ? undefined : params.customer_id, - event_name: params.event_name, + event_names: params.event_name ? [params.event_name] : undefined, limit: params.limit ?? DEFAULT_LIMIT, - cursor_timestamp: params.cursor_timestamp, - cursor_id: params.cursor_id, + offset: 0, }; ctx.logger.debug("Listing raw events via Tinybird pipe", { @@ -126,11 +123,10 @@ export const listRawEvents = async ({ startDate: finalStartDate, endDate: finalEndDate, limit: pipeParams.limit, - hasCursor: !!(params.cursor_timestamp && params.cursor_id), }); const startTime = performance.now(); - const result = await pipes.listEvents(pipeParams); + const result = await pipes.listEventsPaginated(pipeParams); const queryDuration = performance.now() - startTime; ctx.logger.debug("Raw events result", { diff --git a/server/src/internal/events/handlers/handleExternalListEvents.ts b/server/src/internal/events/handlers/handleExternalListEvents.ts index 494c03661..e66ba4724 100644 --- a/server/src/internal/events/handlers/handleExternalListEvents.ts +++ b/server/src/internal/events/handlers/handleExternalListEvents.ts @@ -17,7 +17,7 @@ export const handleExternalListEvents = createRoute({ : [validatedParams.feature_id] : undefined; - const result = await eventActions.listEventsForApi({ + const result = await eventActions.listEvents({ ctx, params: { customer_id: validatedParams.customer_id, diff --git a/server/tinybird/materializations/events_by_timestamp_mv.datasource b/server/tinybird/materializations/events_by_timestamp_mv.datasource index 7c7cb5303..aecf80433 100644 --- a/server/tinybird/materializations/events_by_timestamp_mv.datasource +++ b/server/tinybird/materializations/events_by_timestamp_mv.datasource @@ -1,6 +1,6 @@ DESCRIPTION > Materialized view of events sorted by timestamp for fast time-range queries. - Used by list_events.pipe for the raw logs viewer UI. + Used by list_events_paginated.pipe for the raw logs viewer UI and external API. Sorting key: (org_id, env, timestamp, customer_id, event_name) SCHEMA > diff --git a/server/tinybird/pipes/list_events.pipe b/server/tinybird/pipes/list_events.pipe index dc1903b43..d609f4311 100644 --- a/server/tinybird/pipes/list_events.pipe +++ b/server/tinybird/pipes/list_events.pipe @@ -1,4 +1,6 @@ DESCRIPTION > + DEPRECATED: Use list_events_paginated.pipe instead. + Kept for backwards compatibility during migration. Lists raw events with filtering by org, env, customer, and date range. Optimized for the raw logs viewer UI. Supports pagination via cursor. Queries events_by_timestamp_mv which is sorted by (org_id, env, timestamp) for fast time-range queries. diff --git a/shared/api/common/epochUtils.ts b/shared/api/common/epochUtils.ts new file mode 100644 index 000000000..7b5aa0d92 --- /dev/null +++ b/shared/api/common/epochUtils.ts @@ -0,0 +1,11 @@ +/** Converts epoch ms to ClickHouse DateTime string format */ +export const epochToDateTime = (epochMs: number): string => { + const date = new Date(epochMs); + const year = date.getUTCFullYear(); + const month = String(date.getUTCMonth() + 1).padStart(2, "0"); + const day = String(date.getUTCDate()).padStart(2, "0"); + const hours = String(date.getUTCHours()).padStart(2, "0"); + const minutes = String(date.getUTCMinutes()).padStart(2, "0"); + const seconds = String(date.getUTCSeconds()).padStart(2, "0"); + return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`; +};