diff --git a/apps/docs/mintlify/documentation/getting-started/display-billing.mdx b/apps/docs/mintlify/documentation/getting-started/display-billing.mdx index e9996574d..c8194e5ab 100644 --- a/apps/docs/mintlify/documentation/getting-started/display-billing.mdx +++ b/apps/docs/mintlify/documentation/getting-started/display-billing.mdx @@ -292,7 +292,7 @@ curl -X POST 'https://api.useautumn.com/v1/billing.open_customer_portal' \ ### Usage history chart -Autumn replicates usage data to Clickhouse for aggregate time series queries. Pass the response to a charting library like Recharts. +Autumn provides aggregate time series queries for usage data. Pass the response to a charting library like Recharts. diff --git a/scripts/package.json b/scripts/package.json index fe3c5fd9e..847e31ea0 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -6,12 +6,10 @@ "scripts": { "setup": "tsx setup/setup.js", "setup-test": "tsx setup/setup-test.ts", - "replicate": "bun run db/replicate.ts", - "create-clickhouse-event-views:prod": "infisical run --env=prod -- bun run setup/events/recreate-clickhouse-event-views.ts" + "replicate": "bun run db/replicate.ts" }, "dependencies": { "@autumn/shared": "workspace:*", - "@clickhouse/client": "catalog:", "chalk": "^5.3.0", "dotenv": "^16.5.0", "drizzle-orm": "catalog:", diff --git a/scripts/setup/events/recreate-clickhouse-event-views.ts b/scripts/setup/events/recreate-clickhouse-event-views.ts deleted file mode 100644 index f69efc41d..000000000 --- a/scripts/setup/events/recreate-clickhouse-event-views.ts +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Recreate ClickHouse date range views - * Creates both legacy views (for AnalyticsService) and new event aggregation views (for EventsAggregationService) - * Usage: bun run scripts/setup/events/recreate-clickhouse-event-views.ts - * Or: infisical run --env=dev -- bun scripts/setup/events/recreate-clickhouse-event-views.ts - * Or: infisical run --env=prod -- bun scripts/setup/events/recreate-clickhouse-event-views.ts - */ - -import fs from "node:fs"; -import path from "node:path"; -import { createClient } from "@clickhouse/client"; - -async function main() { - const required = [ - "CLICKHOUSE_URL", - "CLICKHOUSE_USERNAME", - "CLICKHOUSE_PASSWORD", - ]; - - const missing = required.filter((key) => !process.env[key]); - if (missing.length > 0) { - console.error("Missing env vars:", missing.join(", ")); - process.exit(1); - } - - const client = createClient({ - url: process.env.CLICKHOUSE_URL, - username: process.env.CLICKHOUSE_USERNAME, - password: process.env.CLICKHOUSE_PASSWORD, - }); - - try { - const queriesDir = path.join( - import.meta.dir, - "../../../server/src/external/clickhouse/queries", - ); - - const viewsToRecreate = [ - "CREATE_DATE_RANGE_VIEW.sql", - "CREATE_DATE_RANGE_BC_VIEW.sql", - "CREATE_EVENT_AGGREGATION_DATE_RANGE_VIEW.sql", - "CREATE_EVENT_AGGREGATION_DATE_RANGE_BC_VIEW.sql", - ]; - - console.log("šŸ”„ Recreating ClickHouse views...\n"); - - for (const sqlFile of viewsToRecreate) { - const filePath = path.join(queriesDir, sqlFile); - const sql = fs.readFileSync(filePath, "utf8"); - - console.log(`šŸ“ Executing: ${sqlFile}`); - await client.query({ query: sql }); - console.log(`āœ… Success: ${sqlFile}\n`); - } - - console.log("āœ… All views recreated successfully!"); - console.log("\nšŸŽÆ The bin count issue should now be fixed."); - console.log(" Test with your curl command to verify."); - } catch (error) { - console.error("āŒ Error:", error); - process.exit(1); - } finally { - await client.close(); - } -} - -main(); diff --git a/scripts/setup/setup-clickhouse-insights-user.ts b/scripts/setup/setup-clickhouse-insights-user.ts deleted file mode 100644 index 053ce7eaf..000000000 --- a/scripts/setup/setup-clickhouse-insights-user.ts +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Setup readonly user for ClickHouse insights queries - * Usage: bun run scripts/setup-clickhouse-insights-user.ts - * Or: infisical run --env=dev -- bun scripts/setup-clickhouse-insights-user.ts - * Or: infisical run --env=prod -- bun scripts/setup-clickhouse-insights-user.ts - */ - -import crypto from "node:crypto"; -import { createClient } from "@clickhouse/client"; - -const ALLOWED_VIRTUAL_INSIGHTS_TABLES = ["org_events_view"] as const; - -async function main() { - const required = [ - "CLICKHOUSE_URL", - "CLICKHOUSE_USERNAME", - "CLICKHOUSE_PASSWORD", - "CLICKHOUSE_INSIGHTS_USERNAME", - "CLICKHOUSE_INSIGHTS_PASSWORD", - ]; - - const missing = required.filter((key) => !process.env[key]); - if (missing.length > 0) { - console.error("Missing env vars:", missing.join(", ")); - process.exit(1); - } - - const insightsUser = process.env.CLICKHOUSE_INSIGHTS_USERNAME!; - const insightsPassword = process.env.CLICKHOUSE_INSIGHTS_PASSWORD!; - - const client = createClient({ - url: process.env.CLICKHOUSE_URL, - username: process.env.CLICKHOUSE_USERNAME, - password: process.env.CLICKHOUSE_PASSWORD, - }); - - try { - // Check if user exists - const result = await client.query({ - query: "SELECT name FROM system.users WHERE name = {username:String}", - query_params: { username: insightsUser }, - format: "JSONEachRow", - }); - - const users = await result.json(); - - if (users.length === 0) { - // Create user - const passwordHash = crypto - .createHash("sha256") - .update(insightsPassword) - .digest("hex"); - - await client.command({ - query: `CREATE USER ${insightsUser} IDENTIFIED WITH sha256_hash BY '${passwordHash}' SETTINGS readonly = 1`, - }); - console.log(`āœ“ Created user: ${insightsUser}`); - } else { - console.log(`āœ“ User exists: ${insightsUser}`); - } - - // Grant SELECT on virtual tables only - // Note: Views with SQL SECURITY DEFINER execute with creator's privileges, - // so insights_query_user does NOT need access to underlying tables - for (const table of ALLOWED_VIRTUAL_INSIGHTS_TABLES) { - await client.command({ - query: `GRANT SELECT ON ${table} TO ${insightsUser}`, - }); - console.log(`āœ“ Granted SELECT on ${table}`); - } - - console.log("\nāœ… Setup complete"); - } catch (error) { - console.error("āŒ Error:", error); - process.exit(1); - } finally { - await client.close(); - } -} - -main(); diff --git a/server/.env.example b/server/.env.example index ff03b437d..d9a8ad38e 100644 --- a/server/.env.example +++ b/server/.env.example @@ -64,12 +64,3 @@ SVIX_API_KEY= # This is used to generate singular / plural names for your features, which are used when in the dashboard and when you use Autumn UI components ANTHROPIC_API_KEY= -# CLICKHOUSE -# This is used to store your events in a ClickHouse database, which is used for usage analytics -CLICKHOUSE_URL= -CLICKHOUSE_USERNAME= -CLICKHOUSE_PASSWORD= - -# When you first run the server, it needs to create some custom SQL VIEWs and FUNCTIONs, -# Once you run this once you can set this flag to "true" to make the server skip this step, as it only needs to perform this once. -CLICKHOUSE_SKIP_ENSURES= \ No newline at end of file diff --git a/server/src/db/initClickHouse.ts b/server/src/db/initClickHouse.ts deleted file mode 100644 index 3c17e12af..000000000 --- a/server/src/db/initClickHouse.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { type ClickHouseClient, createClient } from "@clickhouse/client"; - -export const clickhouseClient: ClickHouseClient = createClient({ - url: process.env.CLICKHOUSE_URL || undefined, - username: process.env.CLICKHOUSE_USERNAME!, - password: process.env.CLICKHOUSE_PASSWORD!, - max_open_connections: 10, -}); - -export const clickhouse = clickhouseClient; diff --git a/server/src/external/clickhouse/ClickHouseManager.ts b/server/src/external/clickhouse/ClickHouseManager.ts deleted file mode 100644 index 28858218c..000000000 --- a/server/src/external/clickhouse/ClickHouseManager.ts +++ /dev/null @@ -1,246 +0,0 @@ -import fs from "node:fs"; -import path from "node:path"; -import { - type ClickHouseClient, - createClient, - type QueryParams, -} from "@clickhouse/client"; -import { clickhouseClient } from "../../db/initClickHouse.js"; - -export enum ClickHouseQuery { - CREATE_DATE_RANGE_VIEW = "CREATE_DATE_RANGE_VIEW", - CREATE_DATE_RANGE_BC_VIEW = "CREATE_DATE_RANGE_BC_VIEW", - CREATE_ORG_EVENTS_VIEW = "CREATE_ORG_EVENTS_VIEW", - // CREATE_GENERATE_EVENT_COUNT_EXPRESSIONS_FUNCTION = "CREATE_GENERATE_EVENT_COUNTS_EXPRESSIONS", - // CREATE_GENERATE_EVENT_COUNT_EXPRESSIONS_NO_COUNT_FUNCTION = "CREATE_GENERATE_EVENT_COUNTS_EXPRESSIONS_NO_COUNT", - GENERATE_EVENT_COUNT_EXPRESSIONS = "GENERATE_EVENT_COUNT_EXPRESSIONS", - ENSURE_VIEWS_EXIST = "ENSURE_VIEWS_EXIST", - ENSURE_FUNCTIONS_EXIST = "ENSURE_FUNCTIONS_EXIST", -} - -export class ClickHouseManager { - private static instance: ClickHouseManager | null = null; - private client: ClickHouseClient | null = clickhouseClient; - private readonlyClient: ClickHouseClient | null = null; - private initialized = false; - private initPromise: Promise | null = null; - static clickhouseAvailable = - process.env.CLICKHOUSE_URL && - process.env.CLICKHOUSE_USERNAME && - process.env.CLICKHOUSE_PASSWORD; - - private constructor() { - // Empty private constructor - } - - private async initializeClickHouse(): Promise { - console.log("Initializing ClickHouse Manager..."); - console.group(); - if (this.initialized) { - console.log("0. ClickHouse Manager already initialized."); - console.groupEnd(); - return; - } - - console.log("1. Creating ClickHouse client..."); - this.client = clickhouseClient; - - // console.log("2. Checking SQL files exist..."); - // await ClickHouseManager.ensureSQLFilesExist(); - - // console.log("3. Ensuring queries exist..."); - // await this.ensureQueriesExist(); - - console.log("4. ClickHouse Manager initialized."); - console.groupEnd(); - - this.initialized = true; - } - - public static async getInstance(): Promise { - if (!ClickHouseManager.instance) { - ClickHouseManager.instance = new ClickHouseManager(); - ClickHouseManager.instance.initPromise = - ClickHouseManager.instance.initializeClickHouse(); - } - - // Wait for initialization to complete - if (ClickHouseManager.instance.initPromise) { - await ClickHouseManager.instance.initPromise; - } - - return ClickHouseManager.instance; - } - - public static async getClient(): Promise { - const manager = await ClickHouseManager.getInstance(); - if (!manager.client) { - throw new Error("ClickHouse client not initialized"); - } - return manager.client; - } - - public static async getReadonlyClient(): Promise { - const manager = await ClickHouseManager.getInstance(); - if (!manager.readonlyClient) { - if ( - !process.env.CLICKHOUSE_INSIGHTS_USERNAME || - !process.env.CLICKHOUSE_INSIGHTS_PASSWORD - ) { - throw new Error( - "CLICKHOUSE_INSIGHTS_USERNAME and CLICKHOUSE_INSIGHTS_PASSWORD must be set", - ); - } - - manager.readonlyClient = createClient({ - url: process.env.CLICKHOUSE_URL, - username: process.env.CLICKHOUSE_INSIGHTS_USERNAME, - password: process.env.CLICKHOUSE_INSIGHTS_PASSWORD, - }); - } - return manager.readonlyClient; - } - - static async createDateRangeView() {} - static async createDateRangeBcView() {} - static async createOrgEventsView() {} - - static async ensureSQLFilesExist() { - const requiredQueries = [ - ClickHouseQuery.CREATE_DATE_RANGE_VIEW, - ClickHouseQuery.CREATE_DATE_RANGE_BC_VIEW, - ClickHouseQuery.CREATE_ORG_EVENTS_VIEW, - // ClickHouseQuery.CREATE_GENERATE_EVENT_COUNT_EXPRESSIONS_FUNCTION, - // ClickHouseQuery.CREATE_GENERATE_EVENT_COUNT_EXPRESSIONS_NO_COUNT_FUNCTION, - ClickHouseQuery.GENERATE_EVENT_COUNT_EXPRESSIONS, - ClickHouseQuery.ENSURE_VIEWS_EXIST, - ClickHouseQuery.ENSURE_FUNCTIONS_EXIST, - ]; - - const queryResults = await Promise.allSettled( - requiredQueries.map((query) => ClickHouseManager.readSQLFile(query)), - ); - - const failedQueries = queryResults.filter( - (result) => result.status === "rejected", - ); - - if (failedQueries.length > 0) { - console.error( - `Failed to read ${failedQueries.length} ClickHouse queries. Please re-pull the latest version of Autumn. `, - ); - failedQueries.forEach((result, index) => { - if (result.status === "rejected") { - console.error( - `Query ${requiredQueries[index]} failed:`, - result.reason, - ); - } - }); - process.exit(1); - } - } - - // biome-ignore lint/correctness/noUnusedPrivateClassMembers: Might comment this back in in the future - private async ensureQueriesExist() { - if (!this.client) { - throw new Error("ClickHouse client not initialized"); - } - - if (!ClickHouseManager.clickhouseAvailable) { - console.log( - "0. ClickHouse is not available, please set the CLICKHOUSE_URL, CLICKHOUSE_USERNAME, and CLICKHOUSE_PASSWORD environment variables.", - ); - return; - } - - console.log("1. Creating ClickHouse client..."); - this.client = clickhouseClient; - - // Check if we should skip ensuring queries exist - if (process.env.CLICKHOUSE_SKIP_ENSURES?.toLowerCase() === "true") { - console.group(); - console.log( - "āœ“ Skipping query ensures - queries assumed to exist already", - ); - console.groupEnd(); - return; - } - - const queries = [ - ClickHouseQuery.CREATE_DATE_RANGE_BC_VIEW, - ClickHouseQuery.CREATE_DATE_RANGE_VIEW, - ClickHouseQuery.CREATE_ORG_EVENTS_VIEW, - - // ClickHouseQuery.CREATE_GENERATE_EVENT_COUNT_EXPRESSIONS_FUNCTION, - ]; - - console.group(); - - await Promise.all( - queries.map(async (query) => { - try { - await this.executeQuery(query, this.client!); - console.log(`āœ“ Successfully ensured query ${query} exists.`); - } catch (error) { - console.error(`āœ— Failed to execute query ${query}:`, error); - process.exit(1); - } - }), - ); - - console.groupEnd(); - } - - private async readSQLFile(query: ClickHouseQuery) { - const queriesDir = path.join(import.meta.dirname, "queries"); - const queryPath = path.join(queriesDir, `${query}.sql`); - const queryContent = fs.readFileSync(queryPath, "utf8"); - return queryContent; - } - - static async readSQLFile(query: ClickHouseQuery) { - const manager = await ClickHouseManager.getInstance(); - return manager.readSQLFile(query); - } - - private async executeQuery( - query: ClickHouseQuery, - client: ClickHouseClient, - options: any = {}, - ) { - const queryContent = await this.readSQLFile(query); - if (!queryContent) { - throw new Error(`Query ${query} not found`); - } - - // For CREATE FUNCTION queries, use command() instead of query() to avoid FORMAT clause - // if ( - // query === ClickHouseQuery.CREATE_GENERATE_EVENT_COUNT_EXPRESSIONS_FUNCTION - // ) { - // const result = await client.command({ - // query: queryContent, - // ...options, - // }); - // return result; - // } - - const result = await client.query({ - query: queryContent, - ...options, - }); - return result; - } - - static async executeQuery( - query: ClickHouseQuery, - client?: ClickHouseClient, - options: QueryParams = { - format: "TabSeparatedRaw", - } as QueryParams, - ) { - const manager = await ClickHouseManager.getInstance(); - const clickhouseClient = client || (await ClickHouseManager.getClient()); - return manager.executeQuery(query, clickhouseClient, options); - } -} diff --git a/server/src/external/clickhouse/queries/CREATE_DATE_RANGE_BC_VIEW.sql b/server/src/external/clickhouse/queries/CREATE_DATE_RANGE_BC_VIEW.sql deleted file mode 100644 index dd0fcda3d..000000000 --- a/server/src/external/clickhouse/queries/CREATE_DATE_RANGE_BC_VIEW.sql +++ /dev/null @@ -1,17 +0,0 @@ -CREATE or replace VIEW date_range_bc_view AS -SELECT - CASE - WHEN {bin_size:String} = 'hour' THEN - date_trunc('hour', {start_date:DateTime} - interval {days:UInt32} day) + interval number hour - WHEN {bin_size:String} = 'month' THEN - date_trunc('month', {start_date:DateTime} - interval {days:UInt32} day) + interval number month - ELSE - date_trunc('day', {start_date:DateTime} - interval {days:UInt32} day) + interval number day - END as period -FROM numbers( - CASE - WHEN {bin_size:String} = 'hour' THEN {days:UInt32} * 24 + 1 - WHEN {bin_size:String} = 'month' THEN toUInt32(ceil(toFloat64({days:UInt32}) / 30.0)) + 1 - ELSE {days:UInt32} + 1 - END -); diff --git a/server/src/external/clickhouse/queries/CREATE_DATE_RANGE_VIEW.sql b/server/src/external/clickhouse/queries/CREATE_DATE_RANGE_VIEW.sql deleted file mode 100644 index a0cc91441..000000000 --- a/server/src/external/clickhouse/queries/CREATE_DATE_RANGE_VIEW.sql +++ /dev/null @@ -1,17 +0,0 @@ -CREATE or replace VIEW date_range_view AS -SELECT - CASE - WHEN {bin_size:String} = 'hour' THEN - date_trunc('hour', now() - interval {days:UInt32} day) + interval number hour - WHEN {bin_size:String} = 'month' THEN - date_trunc('month', now() - interval {days:UInt32} day) + interval number month - ELSE - date_trunc('day', now() - interval {days:UInt32} day) + interval number day - END as period -FROM numbers( - CASE - WHEN {bin_size:String} = 'hour' THEN {days:UInt32} * 24 + 1 - WHEN {bin_size:String} = 'month' THEN toUInt32(ceil(toFloat64({days:UInt32}) / 30.0)) + 1 - ELSE {days:UInt32} + 1 - END -); diff --git a/server/src/external/clickhouse/queries/CREATE_EVENT_AGGREGATION_DATE_RANGE_BC_VIEW.sql b/server/src/external/clickhouse/queries/CREATE_EVENT_AGGREGATION_DATE_RANGE_BC_VIEW.sql deleted file mode 100644 index 001a3cd85..000000000 --- a/server/src/external/clickhouse/queries/CREATE_EVENT_AGGREGATION_DATE_RANGE_BC_VIEW.sql +++ /dev/null @@ -1,11 +0,0 @@ -CREATE or replace VIEW event_aggregation_date_range_bc_view AS -SELECT - CASE - WHEN {bin_size:String} = 'hour' THEN - date_trunc('hour', {start_date:DateTime} - interval {interval_offset:UInt32} hour) + interval number hour - WHEN {bin_size:String} = 'month' THEN - date_trunc('month', {start_date:DateTime} - interval {interval_offset:UInt32} month) + interval number month - ELSE - date_trunc('day', {start_date:DateTime} - interval {interval_offset:UInt32} day) + interval number day - END as period -FROM numbers({bin_count:UInt32}); diff --git a/server/src/external/clickhouse/queries/CREATE_EVENT_AGGREGATION_DATE_RANGE_VIEW.sql b/server/src/external/clickhouse/queries/CREATE_EVENT_AGGREGATION_DATE_RANGE_VIEW.sql deleted file mode 100644 index 6ee045c1d..000000000 --- a/server/src/external/clickhouse/queries/CREATE_EVENT_AGGREGATION_DATE_RANGE_VIEW.sql +++ /dev/null @@ -1,11 +0,0 @@ -CREATE or replace VIEW event_aggregation_date_range_view AS -SELECT - CASE - WHEN {bin_size:String} = 'hour' THEN - date_trunc('hour', now() - interval {interval_offset:UInt32} hour) + interval number hour - WHEN {bin_size:String} = 'month' THEN - date_trunc('month', now() - interval {interval_offset:UInt32} month) + interval number month - ELSE - date_trunc('day', now() - interval {interval_offset:UInt32} day) + interval number day - END as period -FROM numbers({bin_count:UInt32}); diff --git a/server/src/external/clickhouse/queries/CREATE_GENERATE_EVENT_COUNTS_EXPRESSIONS.sql b/server/src/external/clickhouse/queries/CREATE_GENERATE_EVENT_COUNTS_EXPRESSIONS.sql deleted file mode 100644 index 869aa539c..000000000 --- a/server/src/external/clickhouse/queries/CREATE_GENERATE_EVENT_COUNTS_EXPRESSIONS.sql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE OR REPLACE FUNCTION generateEventCountExpressions AS (event_names) -> -arrayStringConcat( - arrayMap( - event_name -> concat( - 'coalesce(sumIf(e.value, e.event_name = ''', - replaceAll(event_name, '''', ''''''), - '''), 0) as `', - event_name, - '_count`' - ), - event_names - ), - ',\n ' -); \ No newline at end of file diff --git a/server/src/external/clickhouse/queries/CREATE_GENERATE_EVENT_COUNT_EXPRESSIONS_NO_COUNT.sql b/server/src/external/clickhouse/queries/CREATE_GENERATE_EVENT_COUNT_EXPRESSIONS_NO_COUNT.sql deleted file mode 100644 index 71894557a..000000000 --- a/server/src/external/clickhouse/queries/CREATE_GENERATE_EVENT_COUNT_EXPRESSIONS_NO_COUNT.sql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE OR REPLACE FUNCTION generateEventCountExpressionsNoCount AS (event_names) -> -arrayStringConcat( - arrayMap( - event_name -> concat( - 'coalesce(sumIf(e.value, e.event_name = ''', - replaceAll(event_name, '''', ''''''), - '''), 0) as `', - event_name, - '`' - ), - event_names - ), - ',\n' -); \ No newline at end of file diff --git a/server/src/external/clickhouse/queries/CREATE_ORG_EVENTS_VIEW.sql b/server/src/external/clickhouse/queries/CREATE_ORG_EVENTS_VIEW.sql deleted file mode 100644 index 4ae87860f..000000000 --- a/server/src/external/clickhouse/queries/CREATE_ORG_EVENTS_VIEW.sql +++ /dev/null @@ -1,21 +0,0 @@ -CREATE OR REPLACE VIEW org_events_view -SQL SECURITY DEFINER -AS -SELECT - customer_id, - timestamp, - event_name, - case - when isNotNull(JSONExtractString(properties, 'value')) AND JSONExtractString(properties, 'value') != '' - then round(toFloat64OrZero(JSONExtractString(properties, 'value')), 6) - when isNotNull(value) - then round(toFloat64(value), 6) - else 1.0 - end as value, - properties -FROM events -WHERE - set_usage = false - AND env = {env:String} - AND if({org_id:String} != '', org_id = {org_id:String}, true) - AND if({org_slug:String} != '', org_slug = {org_slug:String}, true) \ No newline at end of file diff --git a/server/src/external/clickhouse/queries/ENSURE_FUNCTIONS_EXIST.sql b/server/src/external/clickhouse/queries/ENSURE_FUNCTIONS_EXIST.sql deleted file mode 100644 index 271d7e342..000000000 --- a/server/src/external/clickhouse/queries/ENSURE_FUNCTIONS_EXIST.sql +++ /dev/null @@ -1,6 +0,0 @@ -SELECT - name, - create_query -FROM system.functions -WHERE name = 'generateEventCountExpressions' -AND origin = 'SQLUserDefined' \ No newline at end of file diff --git a/server/src/external/clickhouse/queries/ENSURE_VIEWS_EXIST.sql b/server/src/external/clickhouse/queries/ENSURE_VIEWS_EXIST.sql deleted file mode 100644 index 5002227aa..000000000 --- a/server/src/external/clickhouse/queries/ENSURE_VIEWS_EXIST.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - table_name, - view_definition -FROM INFORMATION_SCHEMA.VIEWS -WHERE table_name = 'date_range_view' -OR table_name = 'date_range_bc_view' -OR table_name = 'org_events_view' -AND table_schema = 'default'; \ No newline at end of file diff --git a/server/src/external/clickhouse/queries/GENERATE_EVENT_COUNT_EXPRESSIONS.sql b/server/src/external/clickhouse/queries/GENERATE_EVENT_COUNT_EXPRESSIONS.sql deleted file mode 100644 index af461ca22..000000000 --- a/server/src/external/clickhouse/queries/GENERATE_EVENT_COUNT_EXPRESSIONS.sql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE OR REPLACE FUNCTION generateEventCountExpressions AS (event_names) -> -arrayStringConcat( - arrayMap( - event_name -> concat( - 'coalesce(sumIf(e.value, e.event_name = ''', - replaceAll(event_name, '''', ''''''), - '''), 0) as ', - replaceRegexpAll(event_name, '[^a-zA-Z0-9]', '_'), - '_count' - ), - event_names - ), - ',\n ' -); \ No newline at end of file diff --git a/server/src/external/tinybird/tinybirdUtils.ts b/server/src/external/tinybird/tinybirdUtils.ts new file mode 100644 index 000000000..40fd7ef9d --- /dev/null +++ b/server/src/external/tinybird/tinybirdUtils.ts @@ -0,0 +1,13 @@ +import { ErrCode, RecaseError } from "@autumn/shared"; +import { StatusCodes } from "http-status-codes"; + +/** Throws SERVICE_UNAVAILABLE if Tinybird is not configured. */ +export const assertTinybirdAvailable = () => { + if (!process.env.TINYBIRD_TOKEN) { + throw new RecaseError({ + message: "Tinybird is not configured, cannot fetch analytics", + code: ErrCode.TinybirdDisabled, + statusCode: StatusCodes.SERVICE_UNAVAILABLE, + }); + } +}; diff --git a/server/src/honoMiddlewares/baseMiddleware.ts b/server/src/honoMiddlewares/baseMiddleware.ts index 11f891827..1f6124eb5 100644 --- a/server/src/honoMiddlewares/baseMiddleware.ts +++ b/server/src/honoMiddlewares/baseMiddleware.ts @@ -7,7 +7,6 @@ import { } from "@autumn/shared"; import type { Context, Next } from "hono"; import { db } from "@/db/initDrizzle.js"; -import { ClickHouseManager } from "@/external/clickhouse/ClickHouseManager.js"; import { logger } from "@/external/logtail/logtailUtils.js"; import type { HonoEnv } from "@/honoUtils/HonoEnv.js"; import { generateId } from "@/utils/genUtils.js"; @@ -15,7 +14,7 @@ import { addRequestToLogs } from "@/utils/logging/addContextToLogs"; /** * Base middleware that sets up the request context - * Sets up: db, logger, clickhouseClient, id, timestamp + * Sets up: db, logger, id, timestamp */ export const baseMiddleware = async (c: Context, next: Next) => { // const env = (c.req.header("app_env") as AppEnv) || AppEnv.Sandbox; @@ -27,8 +26,6 @@ export const baseMiddleware = async (c: Context, next: Next) => { const timestamp = Date.now(); - const clickhouseClient = await ClickHouseManager.getClient(); - const { data: body } = await tryCatch(c.req.json()); const childLogger = addRequestToLogs({ @@ -52,7 +49,6 @@ export const baseMiddleware = async (c: Context, next: Next) => { // Core objects db, logger: childLogger, - clickhouseClient, // Request info id, diff --git a/server/src/honoUtils/HonoEnv.ts b/server/src/honoUtils/HonoEnv.ts index 2a3260391..570b6bb15 100644 --- a/server/src/honoUtils/HonoEnv.ts +++ b/server/src/honoUtils/HonoEnv.ts @@ -5,7 +5,6 @@ import type { Feature, Organization, } from "@autumn/shared"; -import type { ClickHouseClient } from "@clickhouse/client"; import type { User } from "better-auth"; import type { DrizzleCli } from "@/db/initDrizzle.js"; import type { Logger } from "@/external/logtail/logtailUtils.js"; @@ -23,7 +22,6 @@ export type RequestContext = { // Objects db: DrizzleCli; logger: Logger; - clickhouseClient?: ClickHouseClient; // Info id: string; diff --git a/server/src/init.ts b/server/src/init.ts index 41c0451eb..8f4ce4350 100644 --- a/server/src/init.ts +++ b/server/src/init.ts @@ -9,17 +9,11 @@ if (process.env.NODE_ENV !== "development") { import cluster from "node:cluster"; import http from "node:http"; import os from "node:os"; -import { AppEnv } from "@autumn/shared"; -import { toNodeHandler } from "better-auth/node"; -import cors from "cors"; -import express from "express"; -import { client, db } from "./db/initDrizzle.js"; -import { ClickHouseManager } from "./external/clickhouse/ClickHouseManager.js"; +import { getRequestListener } from "@hono/node-server"; +import { client } from "./db/initDrizzle.js"; import { logger } from "./external/logtail/logtailUtils.js"; import { warmupRegionalRedis } from "./external/redis/initRedis.js"; -import { redirectToHono } from "./initHono.js"; -import { auth } from "./utils/auth.js"; -import { generateId } from "./utils/genUtils.js"; +import { createHonoApp } from "./initHono.js"; import { checkEnvVars } from "./utils/initUtils.js"; import { startMemoryMonitor } from "./utils/memoryMonitor.js"; @@ -27,118 +21,20 @@ checkEnvVars(); // subscribeToOrgUpdates({ db }); const init = async () => { - const app = express(); - const server = http.createServer(app); - server.keepAliveTimeout = 120000; // 120 seconds - server.headersTimeout = 120000; // 120 seconds should be >= keepAliveTimeout + const app = createHonoApp(); - app.use(redirectToHono()); - - // Check if this blocks API calls... - const allowedOrigins = [ - "http://localhost:3000", - "http://localhost:5173", - "http://localhost:5174", - "https://app.useautumn.com", - "https://staging.useautumn.com", - "https://dev.useautumn.com", - "https://api.staging.useautumn.com", - "https://localhost:8080", - "https://www.alphalog.ai", - process.env.CLIENT_URL || "", - ]; - - // Wildcard patterns for subdomains - const wildcardPatterns = [ - /^https:\/\/.*\.useautumn\.com$/, - /^https:\/\/.*\.alphalog\.ai$/, - /^https:\/\/.*\.alphalog\.ai$/, - /^chrome-extension:\/\/.*/, - ]; - - app.use( - cors({ - origin: (origin, callback) => { - // Allow requests with no origin (like mobile apps or curl) - if (!origin) { - callback(null, true); - return; - } - - // Check explicit allowed origins - if (allowedOrigins.includes(origin)) { - callback(null, true); - return; - } - - // Check wildcard patterns - if (wildcardPatterns.some((pattern) => pattern.test(origin))) { - callback(null, true); - return; - } - - // Origin not allowed - callback(new Error("Not allowed by CORS")); - }, - credentials: true, - allowedHeaders: [ - "app_env", - "x-api-version", - "x-client-type", - "x-request-id", - "x-visitor-id", - "Authorization", - "Content-Type", - "Accept", - "Origin", - "X-API-Version", - "X-Requested-With", - "Access-Control-Request-Method", - "Access-Control-Request-Headers", - "Cache-Control", - "If-Match", - "If-None-Match", - "If-Modified-Since", - "If-Unmodified-Since", - "User-Agent", // Required for better-auth v1.4.0+ compatibility with Safari/Zen browser - ], - }), - ); - - app.all("/api/auth/*", toNodeHandler(auth)); - - // Initialize managers in parallel for faster startup - await Promise.all([ClickHouseManager.getInstance(), warmupRegionalRedis()]); - - app.use(async (req: any, res: any, next: any) => { - // Add Render region identifier headers for load balancer verification - const serviceName = process.env.RENDER_SERVICE_NAME || "unknown"; - const externalHostname = process.env.RENDER_EXTERNAL_HOSTNAME || "unknown"; - res.setHeader("x-render-service", serviceName); - res.setHeader("x-render-hostname", externalHostname); - - req.env = req.env = req.headers.app_env || AppEnv.Sandbox; - req.db = db; - req.clickhouseClient = await ClickHouseManager.getClient(); - req.id = - req.headers["rndr-id"] || - req.headers["X-Amzn-Trace-Id"] || - req.headers["x-amzn-trace-id"] || - generateId("local_req"); - req.timestamp = Date.now(); - req.expand = []; - req.skipCache = false; - - await next(); - }); - - app.use(express.json()); + await Promise.all([warmupRegionalRedis()]); const PORT = process.env.SERVER_PORT ? Number.parseInt(process.env.SERVER_PORT) : 8080; - // Bind to 0.0.0.0 for AWS ECS/Docker containers + const requestListener = getRequestListener(app.fetch); + const server = http.createServer(requestListener); + + server.keepAliveTimeout = 120000; + server.headersTimeout = 120000; + server.listen(PORT, "0.0.0.0", () => { console.log(`Server running on port ${PORT}`); startMemoryMonitor("server", 60_000); diff --git a/server/src/initHono.ts b/server/src/initHono.ts index e41d67339..f7b111416 100644 --- a/server/src/initHono.ts +++ b/server/src/initHono.ts @@ -3,7 +3,6 @@ import { oauthProviderAuthServerMetadata, oauthProviderOpenIdConfigMetadata, } from "@better-auth/oauth-provider"; -import { getRequestListener } from "@hono/node-server"; import { eq } from "drizzle-orm"; import { Hono } from "hono"; import { cors } from "hono/cors"; @@ -16,9 +15,9 @@ import { errorMiddleware } from "./honoMiddlewares/errorMiddleware.js"; import { traceMiddleware } from "./honoMiddlewares/traceMiddleware.js"; import type { HonoEnv } from "./honoUtils/HonoEnv.js"; import { handleHealthCheck } from "./honoUtils/handleHealthCheck.js"; +import { heapSnapshotRouter } from "./internal/debug/heapSnapshotRoute.js"; import { cliRouter } from "./internal/dev/cli/cliRouter.js"; import { handleOAuthCallback } from "./internal/orgs/handlers/stripeHandlers/handleOAuthCallback.js"; -import { heapSnapshotRouter } from "./internal/debug/heapSnapshotRoute.js"; import { apiRouter } from "./routers/apiRouter.js"; import { internalRouter } from "./routers/internalRouter.js"; import { publicRouter } from "./routers/publicRouter.js"; @@ -66,7 +65,7 @@ const ALLOWED_HEADERS = [ "User-Agent", // Required for better-auth v1.4.0+ compatibility with Safari/Zen browser ]; -const createHonoApp = () => { +export const createHonoApp = () => { const app = new Hono(); // CORS configuration (must be before routes) @@ -158,66 +157,5 @@ const createHonoApp = () => { app.onError(errorMiddleware); - // Create request listener for integration with Express - const requestListener = getRequestListener(app.fetch); - return { honoApp: app, requestListener }; -}; - -/** - * Smart middleware that checks if Hono has a matching route. - * If yes: forwards the request to Hono (fresh, unmodified) - * If no: calls next() to continue Express flow (untouched) - */ -export const redirectToHono = () => { - const { honoApp, requestListener } = createHonoApp(); - - // Get all routes from Hono app - const routes = honoApp.routes; - - return async (req: any, res: any, next: any) => { - const method = req.method; - const path = req.path; - - // Check if Hono has a matching route for this method + path - const hasMatch = routes.some((route) => { - // Check if method matches - if (route.method !== method && route.method !== "ALL") { - return false; - } - - // Check if path matches (handle dynamic routes) - const routePath = route.path; - - // Exact match - if (routePath === path) { - return true; - } - - // Check for wildcard patterns (e.g., /api/autumn/*) - if (routePath.endsWith("/*")) { - const basePath = routePath.slice(0, -2); // Remove "/*" - if (path.startsWith(`${basePath}/`) || path === basePath) { - return true; - } - } - - // Check for dynamic routes (e.g., /v1/products/:id) - if (routePath.includes(":")) { - const routeRegex = new RegExp( - `^${routePath.replace(/:[^/]+/g, "([^/]+)")}$`, - ); - return routeRegex.test(path); - } - - return false; - }); - - if (hasMatch) { - // Route exists in Hono - forward the FRESH request - return requestListener(req, res); - } - - // No match - continue to Express (completely untouched) - next(); - }; + return app; }; diff --git a/server/src/internal/analytics/AnalyticsService.ts b/server/src/internal/analytics/AnalyticsService.ts deleted file mode 100644 index ab7ed000e..000000000 --- a/server/src/internal/analytics/AnalyticsService.ts +++ /dev/null @@ -1,418 +0,0 @@ -import { - ErrCode, - type FullCustomer, - type RangeEnum, - RecaseError, -} from "@autumn/shared"; -import type { ClickHouseClient } from "@clickhouse/client"; -import { Decimal } from "decimal.js"; -import { StatusCodes } from "http-status-codes"; -import type { AutumnContext } from "@/honoUtils/HonoEnv.js"; -import type { ExtendedRequest } from "@/utils/models/Request.js"; -import { - generateEventCountExpressions, - getBillingCycleStartDate, -} from "./analyticsUtils.js"; - -export type TopEventNameRow = { - event_name: string; - count: number; -}; - -export class AnalyticsService { - static clickhouseAvailable = - process.env.CLICKHOUSE_URL && - process.env.CLICKHOUSE_USERNAME && - process.env.CLICKHOUSE_PASSWORD; - - static handleEarlyExit = () => { - if (!AnalyticsService.clickhouseAvailable) { - throw new RecaseError({ - message: "ClickHouse is disabled, cannot fetch events", - code: ErrCode.ClickHouseDisabled, - statusCode: StatusCodes.SERVICE_UNAVAILABLE, - }); - } - }; - - static formatJsDateToClickHouseDateTime(date: Date) { - const year = date.getFullYear(); - const month = String(date.getMonth() + 1).padStart(2, "0"); - const day = String(date.getDate()).padStart(2, "0"); - const hours = String(date.getHours()).padStart(2, "0"); - const minutes = String(date.getMinutes() - 1).padStart(2, "0"); - const seconds = String(date.getSeconds() - 1).padStart(2, "0"); - - return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`; - } - - static async getTopEventNames({ - ctx, - limit = 3, - }: { - ctx: AutumnContext; - limit?: number; - }): Promise<{ - eventNames: string[]; - result: { data: TopEventNameRow[] }; - }> { - const { clickhouseClient, org, env } = ctx; - - if (!clickhouseClient) throw new Error("ClickHouse client not found"); - - const query = ` - select count(*) as count, event_name - from org_events_view(org_id={org_id:String}, org_slug='', env={env:String}) - where timestamp >= NOW() - INTERVAL '1 month' - group by event_name - order by count(*) desc - limit {limit:UInt32} - `; - const result = await clickhouseClient.query({ - query, - query_params: { - org_id: org?.id, - env: env, - limit, - }, - }); - - const resultJson = await result.json(); - - return { - eventNames: resultJson.data.map((row) => row.event_name), - result: resultJson, - }; - } - - static async getTopUser({ req }: { req: ExtendedRequest }) { - const { clickhouseClient, org, env } = req; - - const query = ` -SELECT - c.name -FROM - events e -JOIN - customers c ON e.customer_id = c.id -WHERE - toDate(e.timestamp) = today() - AND e.org_id = {org_id: String} - AND e.env = {env: String} -GROUP BY - c.name -ORDER BY - COUNT(*) DESC -LIMIT 1 - -UNION ALL - -SELECT - 'None' -WHERE - NOT EXISTS ( - SELECT - 1 - FROM - events e - JOIN - customers c ON e.customer_id = c.id - WHERE - toDate(e.timestamp) = today() - AND e.org_id = {org_id: String} - AND e.env = {env: String} - GROUP BY - c.name - ORDER BY - COUNT(*) DESC - LIMIT 1 - ) - `; - - const result = await clickhouseClient.query({ - query, - query_params: { - org_id: org?.id, - env: env, - }, - }); - - const resultJson = await result.json(); - - return (resultJson.data as { name: string; count: number }[])[0]; - } - - static async getTotalEvents({ - ctx, - eventName, - }: { - ctx: AutumnContext; - eventName?: string; - }) { - const { clickhouseClient, org, env } = ctx; - - if (!clickhouseClient) throw new Error("ClickHouse client not found"); - - const query = ` -SELECT SUM( - CASE - WHEN JSONHas(properties, 'value') THEN toInt64(JSONExtractFloat(properties, 'value')) - WHEN value IS NOT NULL THEN toInt64(value) - ELSE 1 - END -) AS total_events -FROM org_events_view(org_id={org_id:String}, org_slug='', env={env:String}) -WHERE event_name = {eventName:String} - `; - - const result = await clickhouseClient.query({ - query, - query_params: { - org_id: org?.id, - env: env, - eventName: eventName ?? undefined, - }, - }); - - const resultJson = await result.json(); - - return (resultJson.data as { total_events: number }[])[0].total_events; - } - - static async getTotalCustomers({ ctx }: { ctx: AutumnContext }) { - const { clickhouseClient, org, env } = ctx; - - if (!clickhouseClient) throw new Error("ClickHouse client not found"); - - const query = `SELECT COUNT(DISTINCT id) AS total_customers -FROM customers -WHERE org_id = {org_id:String} - AND env = {env:String};`; - - const result = await clickhouseClient.query({ - query, - query_params: { - org_id: org?.id, - env: env, - }, - }); - - const resultJson = await result.json(); - - return (resultJson.data as { total_customers: number }[])[0] - .total_customers; - } - - static async getTimeseriesEvents({ - ctx, - params, - customer, - aggregateAll = false, - }: { - ctx: AutumnContext; - params: { - event_names: string[]; - interval: RangeEnum; - customer_id?: string; - no_count?: boolean; - }; - customer?: FullCustomer; - aggregateAll?: boolean; - }) { - const { clickhouseClient, org, env, db } = ctx; - - if (!clickhouseClient) throw new Error("ClickHouse client not found"); - - const intervalType: RangeEnum = params.interval || "24h"; - - const isBillingCycle = intervalType === "1bc" || intervalType === "3bc"; - AnalyticsService.handleEarlyExit(); - - // Skip billing cycle calculation if aggregating all customers - const getBCResults = - isBillingCycle && !aggregateAll && customer - ? ((await getBillingCycleStartDate( - customer, - db, - intervalType as "1bc" | "3bc", - )) as { startDate: string; endDate: string; gap: number } | null) - : null; - - const countExpressions = generateEventCountExpressions( - params.event_names, - params.no_count, - ); - - if (AnalyticsService.clickhouseAvailable) { - const query = ` -with customer_events as ( - select * - from org_events_view(org_id={org_id:String}, org_slug='', env={env:String}) - ${aggregateAll ? "" : "where customer_id = {customer_id:String}"} -) -select - dr.period, - ${countExpressions} -from date_range_view(bin_size={bin_size:String}, days={days:UInt32}) dr - left join customer_events e - on date_trunc({bin_size:String}, e.timestamp) = dr.period -group by dr.period -order by dr.period; -`; - - const queryBillingCycle = ` -with customer_events as ( - select * - from org_events_view(org_id={org_id:String}, org_slug='', env={env:String}) - ${aggregateAll ? "" : "where customer_id = {customer_id:String}"} -) -select - dr.period, - ${countExpressions} -from date_range_bc_view(bin_size={bin_size:String}, start_date={end_date:DateTime}, days={days:UInt32}) dr - left join customer_events e - on date_trunc({bin_size:String}, e.timestamp) = dr.period -group by dr.period -order by dr.period; - `; - - const queryParams = { - org_id: org?.id, - env: env, - customer_id: params.customer_id, - days: - intervalType === "24h" - ? 1 - : intervalType === "7d" - ? 7 - : intervalType === "30d" - ? 30 - : intervalType === "90d" - ? 90 - : intervalType === "1bc" - ? (getBCResults?.gap ?? 0) + 1 - : intervalType === "3bc" - ? (getBCResults?.gap ?? 0) - : 0, - bin_size: intervalType === "24h" ? "hour" : "day", - end_date: isBillingCycle ? getBCResults?.endDate : undefined, - }; - - // Use regular query for aggregateAll or when no billing cycle data is available - const queryToUse = - isBillingCycle && !aggregateAll && getBCResults?.startDate - ? queryBillingCycle - : query; - - const result = await (clickhouseClient as ClickHouseClient).query({ - query: queryToUse, - query_params: queryParams, - format: "JSON", - clickhouse_settings: { - output_format_json_quote_decimals: 0, - output_format_json_quote_64bit_integers: 1, - output_format_json_quote_64bit_floats: 1, - }, - }); - - const resultJson = await result.json(); - - resultJson.data.forEach((row: any) => { - Object.keys(row).forEach((key: string) => { - if (key !== "period") { - row[key] = new Decimal(row[key]).toDecimalPlaces(10).toNumber(); - } - }); - }); - - return resultJson; - } - } - - static async getRawEvents({ - ctx, - params, - customer, - aggregateAll = false, - }: { - ctx: AutumnContext; - params: any; - customer?: FullCustomer; - aggregateAll?: boolean; - }) { - const { clickhouseClient, org, db, env } = ctx; - - if (!clickhouseClient) throw new Error("ClickHouse client not found"); - - AnalyticsService.handleEarlyExit(); - - const startDate = new Date(); - const intervalType = params.interval || "day"; - const isBillingCycle = intervalType === "1bc" || intervalType === "3bc"; - - // Skip billing cycle calculation if aggregating all customers - const getBCResults = - isBillingCycle && !aggregateAll && customer - ? ((await getBillingCycleStartDate( - customer, - db, - intervalType as "1bc" | "3bc", - )) as { startDate: string; endDate: string; gap: number } | null) - : null; - - switch (intervalType) { - case "24h": - startDate.setHours(startDate.getHours() - 24); - break; - case "7d": - startDate.setDate(startDate.getDate() - 7); - break; - case "30d": - startDate.setDate(startDate.getDate() - 30); - break; - case "90d": - startDate.setDate(startDate.getDate() - 90); - break; - default: - startDate.setDate(startDate.getDate() - 24); - break; - } - - const finalStartDate = - isBillingCycle && getBCResults?.startDate - ? getBCResults.startDate - : AnalyticsService.formatJsDateToClickHouseDateTime(startDate); - const finalEndDate = - isBillingCycle && getBCResults?.endDate - ? getBCResults.endDate - : AnalyticsService.formatJsDateToClickHouseDateTime(new Date()); - - const query = ` - SELECT * - FROM org_events_view(org_id={organizationId:String}, org_slug='', env={env:String}) - WHERE timestamp >= toDateTime({startDate:String}) - AND timestamp < toDateTime({endDate:String}) - ${aggregateAll ? "" : "AND customer_id = {customerId:String}"} - ORDER BY timestamp DESC - limit 10000 - `; - - const result = await clickhouseClient.query({ - query: query, - query_params: { - organizationId: org?.id, - customerId: params.customer_id, - startDate: finalStartDate, - endDate: finalEndDate, - env: env, - }, - }); - - // log the actual query... with params filled in...? - // console.log("query", query); - - const resultJson = await result.json(); - - return resultJson; - } -} diff --git a/server/src/internal/analytics/RevenueService.ts b/server/src/internal/analytics/RevenueService.ts deleted file mode 100644 index c64684742..000000000 --- a/server/src/internal/analytics/RevenueService.ts +++ /dev/null @@ -1,42 +0,0 @@ -import type { AutumnContext } from "@/honoUtils/HonoEnv"; - -export class RevenueService { - static clickhouseAvailable = - process.env.CLICKHOUSE_URL && - process.env.CLICKHOUSE_USERNAME && - process.env.CLICKHOUSE_PASSWORD; - - static async getMonthlyRevenue({ ctx }: { ctx: AutumnContext }) { - const { clickhouseClient, org, env } = ctx; - - if (!clickhouseClient) throw new Error("ClickHouse client not found"); - - const query = ` -SELECT - SUM(total) AS total_payment_volume -FROM - invoices -INNER JOIN - customers c ON invoices.internal_customer_id = c.internal_id -WHERE - status = 'paid' - -- Divide by 1000 to convert from milliseconds to seconds, then cast to DateTime - AND toDateTime(CAST(created_at AS Float64) / 1000) >= subtractDays(toStartOfDay(now()), 30) - AND c.org_id = {org_id:String} - AND c.env = {env:String};`; - - const result = await clickhouseClient.query({ - query, - query_params: { - org_id: org?.id, - env: env, - }, - }); - - const resultJson = await result.json(); - - return ( - resultJson.data as { total_payment_volume: number; label: string }[] - )[0]; - } -} diff --git a/server/src/internal/analytics/actions/eventValidationUtils.ts b/server/src/internal/analytics/actions/eventValidationUtils.ts index 959861885..ae467a88c 100644 --- a/server/src/internal/analytics/actions/eventValidationUtils.ts +++ b/server/src/internal/analytics/actions/eventValidationUtils.ts @@ -6,7 +6,7 @@ export const validatePropertyPathForJSON = ({ }: { propertyKey: string; }) => { - // Validate property path segments (matches old ClickHouse behavior) + // Validate property path segments are alphanumeric/underscore only const pathSegments = propertyKey.split("."); for (const segment of pathSegments) { if (!/^[a-zA-Z0-9_]+$/.test(segment)) { diff --git a/server/src/internal/analytics/analyticsUtils.ts b/server/src/internal/analytics/analyticsUtils.ts index 501379ce5..67378114f 100644 --- a/server/src/internal/analytics/analyticsUtils.ts +++ b/server/src/internal/analytics/analyticsUtils.ts @@ -242,7 +242,7 @@ export function generateEventCountExpressions( noCount: boolean = false, ): string { const expressions = eventNames.map((eventName) => { - // Replicate ClickHouse's replaceAll(eventName, '''', '''''') + // Escape single quotes for SQL safety const escapedEventName = eventName.replace(/'/g, "''"); const columnName = noCount ? eventName : `${eventName}_count`; return `coalesce(sumIf(e.value, e.event_name = '${escapedEventName}'), 0) as \`${columnName}\``; diff --git a/server/src/internal/analytics/handlers/handleInsightsQuery.ts b/server/src/internal/analytics/handlers/handleInsightsQuery.ts deleted file mode 100644 index 8db494589..000000000 --- a/server/src/internal/analytics/handlers/handleInsightsQuery.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { - AffectedResource, - applyResponseVersionChanges, - InsightsQueryBodySchema, - type InsightsQueryResponse, - InsightsQueryResponseSchema, -} from "@autumn/shared"; -import { ClickHouseManager } from "../../../external/clickhouse/ClickHouseManager"; -import { createRoute } from "../../../honoMiddlewares/routeHandler"; - -const VIRTUAL_TABLES = ["org_events_view"] as const; - -export const handleInsightsQuery = createRoute({ - body: InsightsQueryBodySchema, - handler: async (c) => { - const ctx = c.get("ctx"); - const { query } = c.req.valid("json"); - - const containsVirtualTable = VIRTUAL_TABLES.some((table) => - query.includes(`from ${table}`), - ); - - if (containsVirtualTable) { - return c.json( - { - data: null, - error: "Virtual table not allowed in query", - }, - 400, - ); - } - - const cleanedQuery = query.replace( - /from\s+events/gi, - `from org_events_view(org_id={org_id:String}, org_slug='', env={env:String})`, - ); - - const readonlyClient = await ClickHouseManager.getReadonlyClient(); - - const result = await readonlyClient.query({ - query: cleanedQuery, - query_params: { - org_id: ctx.org.id, - org_slug: "", - env: ctx.env, - limit: 1000, - }, - format: "JSON", - }); - - const resultJson = await result.json(); - - const parsedResult = InsightsQueryResponseSchema.parse({ - data: resultJson, - }); - - return c.json( - applyResponseVersionChanges({ - input: parsedResult.data, - targetVersion: ctx.apiVersion, - resource: AffectedResource.Attach, - ctx, - }), - ); - }, -}); diff --git a/server/src/internal/analytics/insightsRouter.ts b/server/src/internal/analytics/insightsRouter.ts deleted file mode 100644 index 8e3f3fc5d..000000000 --- a/server/src/internal/analytics/insightsRouter.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { Hono } from "hono"; -import type { HonoEnv } from "../../honoUtils/HonoEnv.js"; -import { handleInsightsQuery } from "./handlers/handleInsightsQuery"; - -export const insightsRouter = new Hono(); - -insightsRouter.post("/query", ...handleInsightsQuery); diff --git a/server/src/internal/analytics/internalHandlers/handleGetEventNames.ts b/server/src/internal/analytics/internalHandlers/handleGetEventNames.ts index 1ade87615..1f4e7a0f4 100644 --- a/server/src/internal/analytics/internalHandlers/handleGetEventNames.ts +++ b/server/src/internal/analytics/internalHandlers/handleGetEventNames.ts @@ -1,6 +1,6 @@ import { type Feature, FeatureType } from "@autumn/shared"; +import { assertTinybirdAvailable } from "@/external/tinybird/tinybirdUtils.js"; import { createRoute } from "@/honoMiddlewares/routeHandler.js"; -import { AnalyticsService } from "../AnalyticsService.js"; import { eventActions } from "../actions/eventActions.js"; /** @@ -8,11 +8,10 @@ import { eventActions } from "../actions/eventActions.js"; */ export const handleGetEventNames = createRoute({ handler: async (c) => { + assertTinybirdAvailable(); const ctx = c.get("ctx"); const { features } = ctx; - AnalyticsService.handleEarlyExit(); - const res = await eventActions.getTopEventNames({ ctx }); const result = res.eventNames; diff --git a/server/src/internal/analytics/internalHandlers/handleInternalAggregateEvents.ts b/server/src/internal/analytics/internalHandlers/handleInternalAggregateEvents.ts index 66442415a..622c9c775 100644 --- a/server/src/internal/analytics/internalHandlers/handleInternalAggregateEvents.ts +++ b/server/src/internal/analytics/internalHandlers/handleInternalAggregateEvents.ts @@ -7,9 +7,9 @@ import { } from "@autumn/shared"; import { StatusCodes } from "http-status-codes"; import { z } from "zod/v4"; +import { assertTinybirdAvailable } from "@/external/tinybird/tinybirdUtils.js"; import { createRoute } from "@/honoMiddlewares/routeHandler.js"; import { CusService } from "@/internal/customers/CusService.js"; -import { AnalyticsService } from "../AnalyticsService.js"; import { eventActions } from "../actions/eventActions.js"; const InternalAggregateEventsSchema = z.object({ @@ -27,14 +27,13 @@ const InternalAggregateEventsSchema = z.object({ export const handleInternalAggregateEvents = createRoute({ body: InternalAggregateEventsSchema, handler: async (c) => { + assertTinybirdAvailable(); const ctx = c.get("ctx"); const { db, org, env, features } = ctx; const { interval, customer_id, group_by, bin_size, timezone } = c.req.valid("json"); let { event_names } = c.req.valid("json"); - AnalyticsService.handleEarlyExit(); - let aggregateAll = false; let customer: FullCustomer | undefined; let bcExclusionFlag = false; diff --git a/server/src/internal/analytics/internalHandlers/handleListEventNames.ts b/server/src/internal/analytics/internalHandlers/handleListEventNames.ts index da2de23c6..71e0d077a 100644 --- a/server/src/internal/analytics/internalHandlers/handleListEventNames.ts +++ b/server/src/internal/analytics/internalHandlers/handleListEventNames.ts @@ -1,6 +1,6 @@ import { z } from "zod/v4"; +import { assertTinybirdAvailable } from "@/external/tinybird/tinybirdUtils.js"; import { createRoute } from "@/honoMiddlewares/routeHandler.js"; -import { AnalyticsService } from "../AnalyticsService.js"; import { eventActions } from "../actions/eventActions.js"; const ListEventNamesSchema = z.object({ @@ -13,11 +13,10 @@ const ListEventNamesSchema = z.object({ export const handleListEventNames = createRoute({ query: ListEventNamesSchema, handler: async (c) => { + assertTinybirdAvailable(); const ctx = c.get("ctx"); const { limit } = c.req.valid("query"); - AnalyticsService.handleEarlyExit(); - const eventNames = await eventActions.listEventNames({ ctx, limit, diff --git a/server/src/internal/customers/cusProducts/cusEnts/cusEntUtils/getExistingUsage.ts b/server/src/internal/customers/cusProducts/cusEnts/cusEntUtils/getExistingUsage.ts index 1f7c2c221..89da81c28 100644 --- a/server/src/internal/customers/cusProducts/cusEnts/cusEntUtils/getExistingUsage.ts +++ b/server/src/internal/customers/cusProducts/cusEnts/cusEntUtils/getExistingUsage.ts @@ -213,11 +213,7 @@ export const addExistingUsagesToCusEnts = ({ let usage = existingUsages[key].usage; const entityUsages = existingUsages[key].entityUsages; - const { - feature_id = "", - interval = "", - interval_count = 1, - } = existingUsages[key] || {}; + const { feature_id = "" } = existingUsages[key] || {}; for (const cusEnt of fullCusEnts) { const ent = cusEnt.entitlement; @@ -273,6 +269,5 @@ export const addExistingUsagesToCusEnts = ({ } } - // console.log("Full cusEnts:", fullCusEnts); return fullCusEnts.map((ce) => CustomerEntitlementSchema.parse(ce)); }; diff --git a/server/src/internal/customers/cusProducts/cusEnts/cusRollovers/getNewProductRollovers.ts b/server/src/internal/customers/cusProducts/cusEnts/cusRollovers/getNewProductRollovers.ts index 22023e9c8..fec73cd24 100644 --- a/server/src/internal/customers/cusProducts/cusEnts/cusRollovers/getNewProductRollovers.ts +++ b/server/src/internal/customers/cusProducts/cusEnts/cusRollovers/getNewProductRollovers.ts @@ -58,52 +58,6 @@ export const getNewProductRollovers = async ({ continue; } - // Bring over current balance (if greater > 0), and any existing rollover - // if ( - // oldCusEnt.balance && - // oldCusEnt.balance > 0 && - // !oldCusEnt.entitlement.entity_feature_id && - // rollover - // ) { - // newRollovers.push({ - // id: generateId("roll"), - // cus_ent_id: newCusEnt.id, - // balance: oldCusEnt.balance, - // entities: {}, - // usage: 0, - // expires_at: calculateNextExpiry(Date.now(), rollover), - // }); - // } else if ( - // oldCusEnt.entitlement.entity_feature_id && - // oldCusEnt.entities - // ) { - // const entityRollovers = Object.keys(oldCusEnt.entities || {}).reduce( - // (acc, entityId) => { - // const entityBalance = oldCusEnt.entities?.[entityId]; - // if (entityBalance && entityBalance.balance > 0) { - // acc[entityId] = { - // id: entityId, - // balance: entityBalance.balance || 0, - // usage: 0, - // }; - // } - // return acc; - // }, - // {} as Record - // ); - - // if (Object.keys(entityRollovers).length > 0) { - // newRollovers.push({ - // id: generateId("roll"), - // cus_ent_id: newCusEnt.id, - // balance: 0, - // entities: entityRollovers, - // usage: 0, - // expires_at: calculateNextExpiry(Date.now(), rollover), - // }); - // } - // } - const curRollovers = oldCusEnt.rollovers; for (const curRollover of curRollovers) { @@ -114,8 +68,6 @@ export const getNewProductRollovers = async ({ }); } - console.log(`Feature ${newEnt?.feature_id} rollovers:`, newRollovers); - // // Add this entitlement's rollover operations rolloverOperations.push({ toInsert: newRollovers, diff --git a/server/src/internal/customers/cusProducts/cusEnts/cusRollovers/rolloverUtils.ts b/server/src/internal/customers/cusProducts/cusEnts/cusRollovers/rolloverUtils.ts index ac59c30c0..27ac15e60 100644 --- a/server/src/internal/customers/cusProducts/cusEnts/cusRollovers/rolloverUtils.ts +++ b/server/src/internal/customers/cusProducts/cusEnts/cusRollovers/rolloverUtils.ts @@ -176,8 +176,6 @@ export function performMaximumClearing({ } }); - // console.log(`id to total:`, entityIdToTotal); - const toUpdate: Rollover[] = []; const toDelete: string[] = []; @@ -191,7 +189,6 @@ export function performMaximumClearing({ const toDeduct = new Decimal(entityTotal).sub(rolloverConfig.max); if (toDeduct.lte(0) || !row.entities[entityId]) continue; - // console.log(`Entity ${entityId}, deducting ${toDeduct.toNumber()}`); const curBalance = new Decimal(row.entities[entityId].balance); let newBalance = curBalance; @@ -216,9 +213,6 @@ export function performMaximumClearing({ }; } } - // console.log(`Max clearing for row ${row.id}`); - // console.log(`Update:`, update.entities); - // If all keys are 0, then delete the row if ( Object.values(update.entities).every( @@ -234,157 +228,3 @@ export function performMaximumClearing({ return { toDelete, toUpdate }; } } - -// For each entity ID, perform maximum clearing... - -// console.log( -// `šŸ” Found ${allEntityIds.size} unique entity IDs: ${Array.from(allEntityIds).join(", ")}` -// ); - -// Sort rows by expiry date (oldest first) -// rows.sort((a, b) => a.expires_at - b.expires_at); -// console.log(`šŸ“… Sorted rows by expiry date (oldest first)`); - -// Track totals per entity ID - -// for (let i = 0; i < rows.length; i++) { -// let row = rows[i]; -// // console.log(`\nšŸ” Processing row ${i + 1}/${rows.length}:`); -// // console.log(` - Row ID: ${row.id}`); -// // console.log(` - Expires at: ${new Date(row.expires_at).toISOString()}`); - -// if (!row.entities || !Array.isArray(row.entities)) { -// console.log(` - āš ļø Row has no entities array, skipping`); -// continue; -// } - -// let rowNeedsUpdate = false; -// let updatedEntities = [...row.entities]; - -// // Process each entity in this row -// for (let j = 0; j < updatedEntities.length; j++) { -// const entity = updatedEntities[j]; -// if (!entity.id || !entity.balance) { -// // console.log(` - āš ļø Entity missing id or balance, skipping`); -// continue; -// } - -// const currentTotal = entityTotals.get(entity.id) || 0; -// const newTotal = currentTotal + entity.balance; - -// console.log( -// ` - Entity ${entity.id}: balance=${entity.balance}, currentTotal=${currentTotal}, newTotal=${newTotal}` -// ); - -// if (newTotal > rolloverConfig.max) { -// const excess = newTotal - rolloverConfig.max; -// const newBalance = entity.balance - excess; - -// console.log( -// ` - āš ļø Total exceeds maximum (${rolloverConfig.max})` -// ); -// console.log(` - Excess to remove: ${excess}`); -// console.log( -// ` - Updating entity balance from ${entity.balance} to ${newBalance}` -// ); - -// if (newBalance > 0) { -// updatedEntities[j] = { ...entity, balance: newBalance }; -// entityTotals.set(entity.id, rolloverConfig.max); -// rowNeedsUpdate = true; -// } else { -// console.log(` - šŸ—‘ļø Removing entity (no remaining balance)`); -// updatedEntities.splice(j, 1); -// j--; // Adjust index after removal -// entityTotals.set(entity.id, rolloverConfig.max); -// rowNeedsUpdate = true; -// } -// } else { -// entityTotals.set(entity.id, newTotal); -// console.log(` - āœ… Total still under maximum, continuing`); -// } -// } - -// // Determine what to do with this row -// if (updatedEntities.length === 0) { -// console.log(` - šŸ—‘ļø Marking row for deletion (no entities remaining)`); -// toDelete.push(row.id); -// } else if (rowNeedsUpdate) { -// console.log(` - āœļø Marking row for update (entities modified)`); -// toUpdate.push({ -// ...row, -// entities: updatedEntities, -// }); -// } else { -// console.log(` - āœ… Row unchanged`); -// } -// } - -// console.log( -// `\nšŸ“‹ Maximum clearing summary for cusEnt ${cusEntID} (entity mode):` -// ); -// console.log(` - Rows to update: ${toUpdate.length}`); -// console.log(` - Rows to delete: ${toDelete.length}`); -// console.log(` - Final entity totals:`); -// entityTotals.forEach((total, entityId) => { -// console.log(` - ${entityId}: ${total}`); -// }); -// if (toUpdate.length > 0) { -// console.log( -// ` - Updated row IDs: ${toUpdate.map((r) => r.id).join(", ")}` -// ); -// } -// if (toDelete.length > 0) { -// console.log(` - Deleted row IDs: ${toDelete.join(", ")}`); -// } -// } - -// return the rows that were cleared - -// for (let i = 0; i < rows.length; i++) { -// let row = rows[i]; -// // console.log(`\nšŸ” Processing row ${i + 1}/${rows.length}:`); -// // console.log(` - Row ID: ${row.id}`); -// // console.log(` - Row balance: ${row.balance}`); -// // console.log(` - Expires at: ${new Date(row.expires_at).toISOString()}`); -// // console.log(` - Total before adding this row: ${total}`); - -// total += row.balance; -// // console.log(` - Total after adding this row: ${total}`); - -// if (total > rolloverConfig.max) { -// let diff = total - rolloverConfig.max; -// // console.log(` - āš ļø Total exceeds maximum (${rolloverConfig.max})`); -// // console.log(` - Difference to remove: ${diff}`); - -// let newBalance = row.balance - diff; -// if (newBalance > 0) { -// // console.log( -// // ` - āœļø Updating row balance from ${row.balance} to ${newBalance}` -// // ); -// toUpdate.push({ -// ...row, -// balance: newBalance, -// }); -// } else { -// // console.log(` - šŸ—‘ļø Marking row for deletion (no remaining balance)`); -// toDelete.push(row.id); -// } -// } else { -// // console.log(` - āœ… Total still under maximum, continuing to next row`); -// continue; -// } -// } - -// console.log(`\nšŸ“‹ Maximum clearing summary for cusEnt ${cusEntID}:`); -// console.log(` - Final total: ${total}`); -// console.log(` - Rows to update: ${toUpdate.length}`); -// console.log(` - Rows to delete: ${toDelete.length}`); -// if (toUpdate.length > 0) { -// console.log( -// ` - Updated balances: ${toUpdate.map((r) => `${r.id}: ${r.balance}`).join(", ")}` -// ); -// } -// if (toDelete.length > 0) { -// console.log(` - Deleted row IDs: ${toDelete.join(", ")}`); -// } diff --git a/server/src/internal/customers/cusProducts/insertCusProduct/initCusEnt/initNextResetAt.ts b/server/src/internal/customers/cusProducts/insertCusProduct/initCusEnt/initNextResetAt.ts index 0aace7e62..452611ecb 100644 --- a/server/src/internal/customers/cusProducts/insertCusProduct/initCusEnt/initNextResetAt.ts +++ b/server/src/internal/customers/cusProducts/insertCusProduct/initCusEnt/initNextResetAt.ts @@ -52,11 +52,6 @@ export const initNextResetAt = ({ freeTrial ?? null, ); - // console.log( - // "Trial end timestamp: ", - // formatUnixToDateTime(trialEndTimestamp! * 1000), - // ); - if (freeTrial && shouldApplyTrial && trialEndTimestamp) { nextResetAtCalculated = new UTCDate(trialEndTimestamp! * 1000); } diff --git a/server/src/internal/events/EventListService.ts b/server/src/internal/events/EventListService.ts deleted file mode 100644 index 9ae4b2c7b..000000000 --- a/server/src/internal/events/EventListService.ts +++ /dev/null @@ -1,196 +0,0 @@ -import type { - ApiEventsListItem, - ApiEventsListParams, - ClickHouseResult, - RawEventFromClickHouse, -} from "@autumn/shared"; -import type { ClickHouseClient } from "@clickhouse/client"; -import type { AutumnContext } from "@/honoUtils/HonoEnv"; - -export class EventListService { - private static transformRawEvents( - rawEvents: RawEventFromClickHouse[], - ): ApiEventsListItem[] { - return rawEvents.map((event) => { - let properties = {}; - if (event.properties) { - try { - properties = JSON.parse(event.properties); - } catch { - // Invalid JSON, use empty object - } - } - - return { - id: event.id, - timestamp: - event.timestamp instanceof Date - ? event.timestamp.getTime() - : typeof event.timestamp === "string" - ? new Date(event.timestamp).getTime() - : Date.now(), - feature_id: event.event_name, - customer_id: event.customer_id, - value: event.value ?? 0, - properties, - }; - }); - } - - private static buildWhereConditions({ - customerId, - eventNames, - startDate, - endDate, - }: { - customerId?: string; - eventNames?: string[]; - startDate?: number; - endDate?: number; - }): string { - const conditions: string[] = []; - - if (customerId) { - conditions.push("customer_id = {customer_id:String}"); - } - - if (eventNames && eventNames.length > 0) { - conditions.push("event_name IN {event_names:Array(String)}"); - } - - if (startDate) { - conditions.push( - "timestamp >= fromUnixTimestamp64Milli({start_date:Int64})", - ); - } - - if (endDate) { - conditions.push( - "timestamp <= fromUnixTimestamp64Milli({end_date:Int64})", - ); - } - - return conditions.length > 0 ? `and ${conditions.join(" and ")}` : ""; - } - - private static buildQueryParams({ - orgId, - env, - customerId, - eventNames, - startDate, - endDate, - offset, - limit, - }: { - orgId: string | undefined; - env: string; - customerId?: string; - eventNames?: string[]; - startDate?: number; - endDate?: number; - offset: number; - limit: number; - }): Record { - const params: Record = { - org_id: orgId, - env, - limit: limit + 1, - offset, - }; - - if (customerId) { - params.customer_id = customerId; - } - - if (eventNames && eventNames.length > 0) { - params.event_names = eventNames; - } - - if (startDate) { - params.start_date = startDate; - } - - if (endDate) { - params.end_date = endDate; - } - - return params; - } - - static async getEvents({ - ctx, - params, - }: { - ctx: AutumnContext; - params: ApiEventsListParams; - }) { - const { clickhouseClient, org, env } = ctx; - const { offset, limit, customer_id, feature_id, custom_range } = params; - - const eventNames = feature_id - ? Array.isArray(feature_id) - ? feature_id - : [feature_id] - : undefined; - - const whereClause = EventListService.buildWhereConditions({ - customerId: customer_id, - eventNames, - startDate: custom_range?.start, - endDate: custom_range?.end, - }); - - const query = ` -select - id, - timestamp, - event_name, - customer_id, - value, - properties -from events -where org_id = {org_id:String} - and env = {env:String} - and set_usage = false - ${whereClause} -order by timestamp desc, id desc -limit {limit:UInt32} -offset {offset:UInt32}; -`; - - const queryParams = EventListService.buildQueryParams({ - orgId: org?.id, - env, - customerId: customer_id, - eventNames, - startDate: custom_range?.start, - endDate: custom_range?.end, - offset, - limit, - }); - - const result = await (clickhouseClient as ClickHouseClient).query({ - query, - query_params: queryParams, - format: "JSON", - }); - - const resultJson = - (await result.json()) as ClickHouseResult; - const rawEvents = resultJson.data; - - const events = EventListService.transformRawEvents(rawEvents); - - const hasMore = events.length > limit; - const list = hasMore ? events.slice(0, limit) : events; - - return { - list, - has_more: hasMore, - total: list.length, - offset, - limit, - }; - } -} diff --git a/server/src/internal/events/EventsAggregationService.ts b/server/src/internal/events/EventsAggregationService.ts deleted file mode 100644 index 1513e0e16..000000000 --- a/server/src/internal/events/EventsAggregationService.ts +++ /dev/null @@ -1,562 +0,0 @@ -import { - BILLING_CYCLE_INTERVALS, - type BillingCycleIntervalEnum, - type BillingCycleResult, - type CalculateCustomRangeParamsInput, - type CalculateCustomRangeParamsOutput, - type CalculateDateRangeParams, - type ClickHouseResult, - type DateRangeResult, - ErrCode, - RecaseError, - type TimeseriesEventsParams, - type TotalEventsParams, -} from "@autumn/shared"; -import type { ClickHouseClient } from "@clickhouse/client"; -import { UTCDate } from "@date-fns/utc"; -import { - add, - differenceInDays, - differenceInHours, - differenceInMonths, - format, - startOfDay, - startOfHour, - startOfMonth, - sub, -} from "date-fns"; -import { Decimal } from "decimal.js"; -import { StatusCodes } from "http-status-codes"; -import type { AutumnContext } from "@/honoUtils/HonoEnv.js"; -import { - generateEventCountExpressions, - getBillingCycleStartDate, -} from "../analytics/analyticsUtils.js"; - -export class EventsAggregationService { - private static dateFormat = "yyyy-MM-dd'T'HH:mm:ss"; - - /** Validates and sanitizes timezone string to prevent SQL injection */ - private static sanitizeTimezone(timezone?: string): string { - if (!timezone) return "UTC"; - - // Only allow valid IANA timezone format (alphanumeric, underscores, slashes, plus/minus) - // Examples: "America/New_York", "Europe/London", "Asia/Ho_Chi_Minh", "UTC", "Etc/GMT+5" - if (!/^[a-zA-Z0-9_/+-]+$/.test(timezone)) { - return "UTC"; - } - - // Limit length to prevent abuse - if (timezone.length > 50) { - return "UTC"; - } - - return timezone; - } - private static async calculateDateRange({ - ctx, - params, - }: { - ctx: AutumnContext; - params: CalculateDateRangeParams; - }): Promise { - const { db } = ctx; - const intervalType = params.interval; - const binSize = - params.bin_size ?? (intervalType === "24h" ? "hour" : "day"); - - if (params.custom_range) { - return { - startDate: format( - new UTCDate(params.custom_range.start), - EventsAggregationService.dateFormat, - ), - endDate: format( - new UTCDate(params.custom_range.end), - EventsAggregationService.dateFormat, - ), - }; - } - - const isBillingCycle = BILLING_CYCLE_INTERVALS.includes( - intervalType as BillingCycleIntervalEnum, - ); - const getBCResults = - isBillingCycle && !params.aggregateAll && params.customer - ? ((await getBillingCycleStartDate( - params.customer, - db, - intervalType as "1bc" | "3bc" | "last_cycle", - )) as BillingCycleResult | null) - : null; - - if (getBCResults?.startDate && getBCResults?.endDate) { - return { - startDate: getBCResults.startDate, - endDate: getBCResults.endDate, - }; - } - - const intervalTypeToDaysMap = - EventsAggregationService.intervalTypeToDaysMap({ - gap: 0, - }); - const days = - intervalTypeToDaysMap[intervalType as keyof typeof intervalTypeToDaysMap]; - - const now = new UTCDate(); - const endDate = format(now, EventsAggregationService.dateFormat); - - const startTime = sub(now, { days }); - const truncatedStartTime = - binSize === "day" ? startOfDay(startTime) : startOfHour(startTime); - const startDate = format( - truncatedStartTime, - EventsAggregationService.dateFormat, - ); - - return { startDate, endDate }; - } - - static intervalTypeToDaysMap({ - gap, - }: { - gap?: number; - } = {}): Record { - return { - "24h": 1, - "7d": 7, - "30d": 30, - "90d": 90, - "1bc": (gap ?? 0) + 1, - "3bc": (gap ?? 0) + 1, - last_cycle: (gap ?? 0) + 1, - }; - } - - private static calculateCustomRangeParams({ - customRange, - binSize, - }: CalculateCustomRangeParamsInput): CalculateCustomRangeParamsOutput { - const startDate = new UTCDate(customRange.start); - const endDate = new UTCDate(customRange.end); - - const filterStartDate = format( - startDate, - EventsAggregationService.dateFormat, - ); - const filterEndDate = format(endDate, EventsAggregationService.dateFormat); - - if (binSize === "hour") { - const truncStart = startOfHour(startDate); - const truncEnd = startOfHour(endDate); - const endPlusOne = add(truncEnd, { hours: 1 }); - const hours = differenceInHours(endPlusOne, truncStart); - - return { - binCount: hours, - binEndDate: format(endPlusOne, EventsAggregationService.dateFormat), - filterStartDate, - filterEndDate, - }; - } - - if (binSize === "month") { - const truncStart = startOfMonth(startDate); - const truncEnd = startOfMonth(endDate); - const endPlusOne = add(truncEnd, { months: 1 }); - const months = differenceInMonths(endPlusOne, truncStart); - - return { - binCount: months, - binEndDate: format(endPlusOne, EventsAggregationService.dateFormat), - filterStartDate, - filterEndDate, - }; - } - - const truncStart = startOfDay(startDate); - const truncEnd = startOfDay(endDate); - const endPlusOne = add(truncEnd, { days: 1 }); - - return { - binCount: differenceInDays(endPlusOne, truncStart), - binEndDate: format(endPlusOne, EventsAggregationService.dateFormat), - filterStartDate, - filterEndDate, - }; - } - - static async getTimeseriesEvents({ - ctx, - params, - }: { - ctx: AutumnContext; - params: TimeseriesEventsParams; - }) { - const { clickhouseClient, org, env, db } = ctx; - - const intervalType = params.interval; - - const useCustomDateQuery = - BILLING_CYCLE_INTERVALS.includes( - intervalType as BillingCycleIntervalEnum, - ) || !!params.custom_range; - - const shouldCalculateBillingCycle = - useCustomDateQuery && - !params.aggregateAll && - params.customer && - !params.custom_range; - - const getBCResults = shouldCalculateBillingCycle - ? ((await getBillingCycleStartDate( - params.customer, - db, - intervalType as "1bc" | "3bc" | "last_cycle", - )) as BillingCycleResult | null) - : null; - - const countExpressions = generateEventCountExpressions( - params.event_names, - params.no_count, - ); - - const getGroupByClause = () => { - if (!params.group_by) - return { select: "", groupBy: "", orderBy: "", fieldName: null }; - - let field: string | null = null; - const propertyPath = params.group_by.replace("properties.", ""); - const pathSegments = propertyPath.split(".").map((segment) => { - // Validate each segment contains only safe characters (alphanumeric, underscores) - if (!/^[a-zA-Z0-9_]+$/.test(segment)) { - throw new RecaseError({ - message: - "Invalid property path. Should only contain alphanumeric and underscore characters.", - code: ErrCode.InvalidInputs, - statusCode: StatusCodes.BAD_REQUEST, - }); - } - // Escape single quotes for SQL safety - return segment.replace(/'/g, "''"); - }); - - const validSegments = pathSegments.filter( - (segment): segment is string => segment !== null, - ); - - if (validSegments.length === 0) { - return { select: "", groupBy: "", orderBy: "", fieldName: null }; - } - - const escapedPathArgs = validSegments.map((seg) => `'${seg}'`).join(", "); - field = `JSONExtractString(e.properties, ${escapedPathArgs})`; - - if (!field) - return { select: "", groupBy: "", orderBy: "", fieldName: null }; - - const escapedFieldName = params.group_by.replace(/`/g, "``"); - const columnAlias = `\`${escapedFieldName}\``; - - return { - select: `, ${field} as ${columnAlias}`, - groupBy: `, ${field}`, - orderBy: `, ${field}`, - fieldName: params.group_by, - }; - }; - - const groupBy = getGroupByClause(); - const groupByFieldName = groupBy.fieldName; - - // Sanitize timezone parameter - const timezone = EventsAggregationService.sanitizeTimezone(params.timezone); - - // Validate distinct group count if group_by is provided - if (params.group_by && groupBy.groupBy) { - const propertyPath = params.group_by.replace("properties.", ""); - const pathSegments = propertyPath.split("."); - const escapedPathArgs = pathSegments - .map((seg) => `'${seg.replace(/'/g, "''")}'`) - .join(", "); - const groupField = `JSONExtractString(e.properties, ${escapedPathArgs})`; - - const distinctCountQuery = ` - SELECT COUNT(DISTINCT ${groupField}) as distinct_count - FROM org_events_view(org_id={org_id:String}, org_slug='', env={env:String}) e - ${params.aggregateAll ? "" : "WHERE e.customer_id = {customer_id:String}"} - `; - - const distinctResult = await (clickhouseClient as ClickHouseClient).query( - { - query: distinctCountQuery, - query_params: { - org_id: org?.id, - env: env, - customer_id: params.customer_id, - }, - format: "JSON", - }, - ); - - const distinctJson = (await distinctResult.json()) as ClickHouseResult<{ - distinct_count: string; - }>; - const distinctCount = Number(distinctJson.data[0]?.distinct_count ?? 0); - - if (distinctCount > 100) { - throw new RecaseError({ - message: `Too many distinct group values (${distinctCount}). Maximum allowed is 100. Please choose a property with fewer unique values.`, - code: ErrCode.InvalidInputs, - statusCode: StatusCodes.BAD_REQUEST, - }); - } - } - - const query = ` -with date_range as ( - select - CASE - WHEN {bin_size:String} = 'hour' THEN - date_trunc('hour', now(), {timezone:String}) - interval {interval_offset:UInt32} hour + interval number hour - WHEN {bin_size:String} = 'month' THEN - date_trunc('month', now(), {timezone:String}) - interval {interval_offset:UInt32} month + interval number month - ELSE - date_trunc('day', now(), {timezone:String}) - interval {interval_offset:UInt32} day + interval number day - END as period - from numbers({bin_count:UInt32}) -), -customer_events as ( - select * - from org_events_view(org_id={org_id:String}, org_slug='', env={env:String}) - ${params.aggregateAll ? "" : "where customer_id = {customer_id:String}"} -) -select - dr.period${groupBy.select}, - ${countExpressions} -from date_range dr - left join customer_events e - on date_trunc({bin_size:String}, e.timestamp, {timezone:String}) = dr.period -group by dr.period${groupBy.groupBy} -order by dr.period${groupBy.orderBy}; -`; - - const customRangeFilter = params.custom_range - ? "and e.timestamp >= {filter_start_date:DateTime} and e.timestamp <= {filter_end_date:DateTime}" - : ""; - - const queryBillingCycle = ` -with date_range as ( - select - CASE - WHEN {bin_size:String} = 'hour' THEN - date_trunc('hour', {end_date:DateTime}, {timezone:String}) - interval {interval_offset:UInt32} hour + interval number hour - WHEN {bin_size:String} = 'month' THEN - date_trunc('month', {end_date:DateTime}, {timezone:String}) - interval {interval_offset:UInt32} month + interval number month - ELSE - date_trunc('day', {end_date:DateTime}, {timezone:String}) - interval {interval_offset:UInt32} day + interval number day - END as period - from numbers({bin_count:UInt32}) -), -customer_events as ( - select * - from org_events_view(org_id={org_id:String}, org_slug='', env={env:String}) - ${params.aggregateAll ? "" : "where customer_id = {customer_id:String}"} -) -select - dr.period${groupBy.select}, - ${countExpressions} -from date_range dr - left join customer_events e - on date_trunc({bin_size:String}, e.timestamp, {timezone:String}) = dr.period - ${customRangeFilter} -group by dr.period${groupBy.groupBy} -order by dr.period${groupBy.orderBy}; - `; - - const { binCount, binEndDate, filterStartDate, filterEndDate } = - params.custom_range - ? EventsAggregationService.calculateCustomRangeParams({ - customRange: params.custom_range, - binSize: params.bin_size, - }) - : { - binCount: undefined, - binEndDate: undefined, - filterStartDate: undefined, - filterEndDate: undefined, - }; - - const intervalTypeToDaysMap = - EventsAggregationService.intervalTypeToDaysMap({ - gap: getBCResults?.gap, - }); - - const binSize = - params.bin_size ?? (intervalType === "24h" ? "hour" : "day"); - - const currentDayOffset = 1; - const calculateBinCount = (days: number): number => { - if (binSize === "hour") { - return days * 24 + currentDayOffset; - } - if (binSize === "month") { - // Convert days to months (approximate: 30 days per month) - return Math.ceil(days / 30) + currentDayOffset; - } - return days + currentDayOffset; - }; - - const standardIntervalBinCount = - intervalTypeToDaysMap[intervalType as keyof typeof intervalTypeToDaysMap]; - - // Billing cycles already have correct count (gap + 1), don't add another offset - const isBillingCycle = - BILLING_CYCLE_INTERVALS.includes( - intervalType as BillingCycleIntervalEnum, - ) && - !params.custom_range && - getBCResults?.gap !== undefined; - - // Multiplier to convert from days to the appropriate bin size unit - const binMultiplier = - binSize === "hour" ? 24 : binSize === "month" ? 1 / 30 : 1; - - const finalBinCount = - binCount ?? - (isBillingCycle - ? Math.ceil(standardIntervalBinCount * binMultiplier) - : calculateBinCount(standardIntervalBinCount)); - - // Use date_range_bc_view query for billing cycles or custom ranges - const useBillingCycleQuery = - useCustomDateQuery && - !params.aggregateAll && - (getBCResults?.startDate || params.custom_range); - - const queryToUse = useBillingCycleQuery ? queryBillingCycle : query; - - // Calculate interval offset based on query type: - // - Billing cycles: offset = gap (how far back from end_date to start_date) - // - Custom ranges: offset = bin_count (already calculated correctly with +1) - // - Standard intervals: offset = bin_count - 1 (to include current period) - let intervalOffset: number; - if (isBillingCycle) { - intervalOffset = Math.ceil(getBCResults.gap * binMultiplier); - } else if (useBillingCycleQuery) { - intervalOffset = finalBinCount; - } else { - intervalOffset = finalBinCount - 1; - } - - const queryParams = { - org_id: org?.id, - env: env, - customer_id: params.customer_id, - bin_count: finalBinCount, - interval_offset: intervalOffset, - bin_size: binSize, - end_date: binEndDate ?? getBCResults?.endDate, - filter_start_date: filterStartDate, - filter_end_date: filterEndDate, - timezone, - }; - - const result = await (clickhouseClient as ClickHouseClient).query({ - query: queryToUse, - query_params: queryParams, - format: "JSON", - clickhouse_settings: { - output_format_json_quote_decimals: 0, - output_format_json_quote_64bit_integers: 1, - output_format_json_quote_64bit_floats: 1, - }, - }); - - const resultJson = (await result.json()) as ClickHouseResult; - - resultJson.data.forEach((row) => { - Object.keys(row).forEach((key: string) => { - // Don't convert period or the group_by field to decimal - if (key !== "period" && key !== groupByFieldName) { - row[key] = new Decimal(row[key] as string | number) - .toDecimalPlaces(10) - .toNumber(); - } - }); - }); - - return resultJson; - } - - static async getTotalEvents({ - ctx, - params, - }: { - ctx: AutumnContext; - params: TotalEventsParams; - }) { - const { clickhouseClient, org, env } = ctx; - - const { startDate, endDate } = - await EventsAggregationService.calculateDateRange({ - ctx, - params: { - interval: params.interval, - bin_size: params.bin_size, - custom_range: params.custom_range, - customer: params.customer, - aggregateAll: params.aggregateAll, - }, - }); - - const query = ` -with customer_events as ( - select * - from org_events_view(org_id={org_id:String}, org_slug='', env={env:String}) - ${params.aggregateAll ? "" : "where customer_id = {customer_id:String}"} -) -select - e.event_name, - COUNT(*) as count, - SUM(e.value) as sum -from customer_events e -where e.timestamp >= {start_date:DateTime} - and e.timestamp <= {end_date:DateTime} - and e.event_name IN {event_names:Array(String)} -group by e.event_name; -`; - - const result = await (clickhouseClient as ClickHouseClient).query({ - query, - query_params: { - org_id: org?.id, - env: env, - customer_id: params.customer_id, - start_date: startDate, - end_date: endDate, - event_names: params.event_names, - }, - format: "JSON", - }); - - const resultJson = (await result.json()) as ClickHouseResult; - const rows = resultJson.data as Array<{ - event_name: string; - count: string; - sum: string; - }>; - - return rows.reduce( - (acc, row) => { - acc[row.event_name] = { - count: new Decimal(row.count).toDecimalPlaces(10).toNumber(), - sum: new Decimal(row.sum ?? 0).toDecimalPlaces(10).toNumber(), - }; - return acc; - }, - {} as Record, - ); - } -} diff --git a/server/src/internal/events/handlers/handleExternalListEvents.ts b/server/src/internal/events/handlers/handleExternalListEvents.ts index 0d5824cd0..e66ba4724 100644 --- a/server/src/internal/events/handlers/handleExternalListEvents.ts +++ b/server/src/internal/events/handlers/handleExternalListEvents.ts @@ -11,8 +11,6 @@ export const handleExternalListEvents = createRoute({ c.req.valid("json"), ); - console.log("Validated params", validatedParams); - const featureIds = validatedParams.feature_id ? Array.isArray(validatedParams.feature_id) ? validatedParams.feature_id diff --git a/server/src/internal/misc/trmnl/handlers/handleGenerateTrmnlScreen.ts b/server/src/internal/misc/trmnl/handlers/handleGenerateTrmnlScreen.ts index b1ea29baf..16c18f225 100644 --- a/server/src/internal/misc/trmnl/handlers/handleGenerateTrmnlScreen.ts +++ b/server/src/internal/misc/trmnl/handlers/handleGenerateTrmnlScreen.ts @@ -1,7 +1,6 @@ import { type Feature, getFeatureName, RecaseError } from "@autumn/shared"; import { createRoute } from "@/honoMiddlewares/routeHandler.js"; -import { AnalyticsService } from "@/internal/analytics/AnalyticsService.js"; -import { RevenueService } from "@/internal/analytics/RevenueService.js"; +import { eventActions } from "@/internal/analytics/actions/eventActions.js"; function numberWithCommas(x: number | string) { return x.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ","); @@ -15,7 +14,7 @@ export const handleGenerateTrmnlScreen = createRoute({ const ctx = c.get("ctx"); const { org, features } = ctx; - const { result } = await AnalyticsService.getTopEventNames({ + const { result } = await eventActions.getTopEventNames({ ctx, limit: 1, }); @@ -38,57 +37,49 @@ export const handleGenerateTrmnlScreen = createRoute({ plural: true, }); - let totalEvents: number | string = await AnalyticsService.getTotalEvents({ - ctx, - eventName: topEvent, - }); - - if (!totalEvents) { - totalEvents = "Unknown"; - } - - let monthlyRevenue: - | { total_payment_volume: number; label: string } - | string = await RevenueService.getMonthlyRevenue({ - ctx, - }); - - if (!monthlyRevenue) { - monthlyRevenue = { - total_payment_volume: 0, - label: "Unknown", - }; - } - - let results = await AnalyticsService.getTimeseriesEvents({ + // Get total event count for the top event + const countAndSum = await eventActions.getCountAndSum({ ctx, params: { event_names: [topEvent], interval: "30d", + aggregateAll: true, + bin_size: "day", }, - aggregateAll: true, }); - let totalCustomers: number | string = - await AnalyticsService.getTotalCustomers({ - ctx, - }); - - if (!totalCustomers) { - totalCustomers = "Unknown"; + let totalEvents: number | string = countAndSum[topEvent]?.count ?? 0; + if (!totalEvents) { + totalEvents = "Unknown"; } - if (!results?.data) { - results = { - data: [], - }; - } + // Revenue: stubbed to 0 for now (Tinybird only stores events today) + const monthlyRevenue = { + total_payment_volume: 0, + label: "N/A", + }; + + // Get timeseries data for the chart + const { formatted: results } = await eventActions.aggregate({ + ctx, + params: { + event_names: [topEvent], + interval: "30d", + aggregateAll: true, + bin_size: "day", + }, + }); + + // Total customers: stubbed to 0 for now (Tinybird only stores events today) + const totalCustomers: number | string = 0; + + const data = results?.data ?? []; // Access hideRevenue from the trmnl context set by middleware const hideRevenue = (org as any).hideRevenue ?? false; return c.json({ - rowData: `[${results.data.map((row: any) => `['${row.period}', ${row[`${topEvent}_count`]}]`).join(",")}]`, + rowData: `[${data.map((row: any) => `['${row.period}', ${row[`${topEvent}_count`]}]`).join(",")}]`, revenue: numberWithCommas(monthlyRevenue.total_payment_volume), totalEvents: numberWithCommas(totalEvents), totalCustomers: numberWithCommas(totalCustomers), diff --git a/server/src/internal/products/prices/priceUtils.ts b/server/src/internal/products/prices/priceUtils.ts index 6e8187da6..8752d2342 100644 --- a/server/src/internal/products/prices/priceUtils.ts +++ b/server/src/internal/products/prices/priceUtils.ts @@ -17,11 +17,7 @@ import { type UsagePriceConfig, } from "@autumn/shared"; import RecaseError from "@server/utils/errorUtils.js"; -import { - compareObjects, - generateId, - notNullish, -} from "@server/utils/genUtils.js"; +import { generateId, notNullish } from "@server/utils/genUtils.js"; import { Decimal } from "decimal.js"; import { StatusCodes } from "http-status-codes"; import { compareBillingIntervals } from "./priceUtils/priceIntervalUtils.js"; @@ -232,42 +228,6 @@ export const getPriceOptions = ( return options; }; -const pricesAreSame = (price1: Price, price2: Price) => { - for (const key in price1.config) { - const originalValue = (price1.config as any)[key]; - const newValue = (price2.config as any)[key]; - - if (key === "usage_tiers") { - for (let i = 0; i < originalValue.length; i++) { - const originalTier = originalValue[i]; - const newTier = newValue[i]; - if (!compareObjects(originalTier, newTier)) { - return false; - } - } - } else if (originalValue !== newValue) { - return false; - } - } - - return true; -}; - -const getUsageTier = (price: Price, quantity: number) => { - const usageConfig = price.config as UsagePriceConfig; - for (let i = 0; i < usageConfig.usage_tiers.length; i++) { - if (i === usageConfig.usage_tiers.length - 1) { - return usageConfig.usage_tiers[i]; - } - - const tier = usageConfig.usage_tiers[i]; - if (tier.to === TierInfinite || tier.to >= quantity) { - return tier; - } - } - return usageConfig.usage_tiers[0]; -}; - export const getPriceForOverage = (price: Price, overage?: number) => { const usageConfig = price.config as UsagePriceConfig; const billingType = getBillingType(usageConfig); diff --git a/server/src/queue/createWorkerContext.ts b/server/src/queue/createWorkerContext.ts index 65bdb159c..918e35b87 100644 --- a/server/src/queue/createWorkerContext.ts +++ b/server/src/queue/createWorkerContext.ts @@ -62,7 +62,6 @@ export const createWorkerContext = async ({ isPublic: false, authType: AuthType.Worker, apiVersion, - clickhouseClient: undefined, expand: [], skipCache: true, extraLogs: {}, diff --git a/server/src/routers/apiRouter.ts b/server/src/routers/apiRouter.ts index 1b6596982..5cda1a562 100644 --- a/server/src/routers/apiRouter.ts +++ b/server/src/routers/apiRouter.ts @@ -1,5 +1,4 @@ import { Hono } from "hono"; -import { insightsRouter } from "@/internal/analytics/insightsRouter.js"; import { legacyAnalyticsRouter } from "@/internal/analytics/legacyAnalyticsRouter.js"; import { eventsRouter } from "@/internal/events/eventsRouter.js"; import { componentsRouter } from "@/internal/misc/components/componentsRouter.js"; @@ -73,7 +72,6 @@ apiRouter.route("/rewards", rewardRouter); apiRouter.route("/reward_programs", rewardProgramRouter); apiRouter.route("/referrals", referralRouter); apiRouter.route("/redemptions", redemptionRouter); -apiRouter.route("/insights", insightsRouter); apiRouter.route("/query", legacyAnalyticsRouter); apiRouter.route("/events", eventsRouter); diff --git a/server/src/utils/errorUtils.ts b/server/src/utils/errorUtils.ts index 04aeb55b2..e5846e4d9 100644 --- a/server/src/utils/errorUtils.ts +++ b/server/src/utils/errorUtils.ts @@ -1,17 +1,4 @@ -import { ErrCode } from "@autumn/shared"; -import * as Sentry from "@sentry/bun"; import chalk from "chalk"; -import { StatusCodes } from "http-status-codes"; -import Stripe from "stripe"; -import { ZodError } from "zod/v4"; -import { formatZodError } from "../errors/formatZodError.js"; -import { getSentryTags } from "../external/sentry/sentryUtils.js"; - -const isPaymentDeclined = (error: any) => { - return ( - error instanceof RecaseError && error.code === ErrCode.StripeCardDeclined - ); -}; export default class RecaseError extends Error { code: string; @@ -48,138 +35,3 @@ export default class RecaseError extends Error { } } } - -export const handleRequestError = ({ - error, - req, - res, - action, -}: { - error: any; - req: any; - res: any; - action: string; -}) => { - try { - Sentry.captureException(error, { - tags: getSentryTags({ - ctx: req, - path: req.originalUrl, - method: req.method, - }), - }); - - const logger = req.logger; - if (error instanceof RecaseError) { - logger.warn( - `RECASE WARNING (${req.org?.slug || "unknown"}): ${error.message} [${error.code}]`, - { - error: error.data ?? error, - }, - ); - - res.status(error.statusCode).json({ - message: error.message, - code: error.code, - env: req.env, - }); - return; - } - - if (error instanceof Stripe.errors.StripeError) { - let curStack; - try { - throw new Error("test"); - } catch (e: any) { - curStack = e.stack; - } - - const { raw, headers, ...rest } = error; - logger.error( - `STRIPE ERROR (${req.org?.slug || "unknown"}): ${error.message}`, - { - error: { - ...rest, - stack: curStack, - }, - }, - ); - - res.status(400).json({ - message: `(Stripe Error) ${error.message}`, - code: `stripe_error`, - }); - } else if (error instanceof ZodError) { - logger.error( - `ZOD ERROR (${req.org?.slug || "unknown"}): ${formatZodError(error)}`, - ); - - res.status(400).json({ - message: formatZodError(error), - code: ErrCode.InvalidInputs, - }); - } else { - logger.error( - `UNKNOWN ERROR (${req.org?.slug || "unknown"}): ${error.message}, ${error.stack}`, - { - error: { - stack: error.stack, - message: error.message, - }, - }, - ); - - res.status(500).json({ - message: error.message || "Unknown error", - code: error.code || "unknown_error", - }); - } - } catch (error) { - console.log("Failed to log error / warning"); - console.log(`Request: ${req.originalUrl}`); - console.log(`Body: ${req.body}`); - console.log(`Log Error: ${error}`); - } -}; - -const handleFrontendReqError = ({ - error, - req, - res, - action, -}: { - error: any; - req: any; - res: any; - action: string; -}) => { - try { - const logger = req.logger; - if ( - error instanceof RecaseError && - error.statusCode === StatusCodes.NOT_FOUND - ) { - // Temporarily disable logger to prevent thread-stream crashes - console.log(`(frontend) ${req.method} ${req.originalUrl}: not found`); - res.status(404).json({ - message: error.message, - code: error.code, - }); - return; - } - - logger.error( - `(frontend) ${req.method} ${req.originalUrl}: ${error.message}`, - { - error, - }, - ); - - res.status(400).json({ - message: error.message || "Unknown error", - code: error.code || "unknown_error", - }); - } catch (error) { - console.log("Failed to log error / warning"); - } -}; diff --git a/server/src/utils/genUtils.ts b/server/src/utils/genUtils.ts index 9da39fd60..6379d72da 100644 --- a/server/src/utils/genUtils.ts +++ b/server/src/utils/genUtils.ts @@ -12,36 +12,12 @@ export const generateId = (prefix: string) => { } }; -export const compareObjects = (obj1: any, obj2: any) => { - for (const key in obj1) { - if (nullish(obj1[key]) && nullish(obj2[key])) { - continue; - } - - if (nullish(obj1[key]) || nullish(obj2[key])) { - return false; - } - - if (obj1[key] !== obj2[key]) { - console.log("Key", key); - console.log("Obj1", obj1[key]); - console.log("Obj2", obj2[key]); - return false; - } - } - return true; -}; - export const keyToTitle = (key: string) => { return key .replace(/[-_]/g, " ") .replace(/\b\w/g, (char) => char.toUpperCase()); }; -const notNullOrUndefined = (value: T | null | undefined): value is T => { - return value !== null && value !== undefined; -}; - export const nullOrUndefined = (value: T | null | undefined): value is T => { return value === null || value === undefined; }; diff --git a/server/src/utils/initUtils.ts b/server/src/utils/initUtils.ts index 0b9a6c669..05b087007 100644 --- a/server/src/utils/initUtils.ts +++ b/server/src/utils/initUtils.ts @@ -36,16 +36,6 @@ export const checkEnvVars = () => { ); } - if ( - !process.env.CLICKHOUSE_URL || - !process.env.CLICKHOUSE_USERNAME || - !process.env.CLICKHOUSE_PASSWORD - ) { - logger.warn( - `CLICKHOUSE_URL or CLICKHOUSE_USERNAME or CLICKHOUSE_PASSWORD is not set, some actions will be skipped`, - ); - } - if (!process.env.SVIX_API_KEY) { logger.warn(`SVIX_API_KEY is not set, some actions will be skipped`); return; diff --git a/server/src/utils/models/Request.ts b/server/src/utils/models/Request.ts index af23ecd7b..26879c569 100644 --- a/server/src/utils/models/Request.ts +++ b/server/src/utils/models/Request.ts @@ -5,7 +5,6 @@ import type { Feature, Organization, } from "@autumn/shared"; -import type { ClickHouseClient } from "@clickhouse/client"; import type { Request as ExpressRequest, Response as ExpressResponse, @@ -21,8 +20,6 @@ export interface ExtendedRequest extends ExpressRequest { db: DrizzleCli; logtail: Logger; logger: Logger; - clickhouseClient: ClickHouseClient; - id?: string; userId?: string; isPublic?: boolean; @@ -35,13 +32,4 @@ export interface ExtendedRequest extends ExpressRequest { skipCache: boolean; } -interface ActionRequest { - id: string; - authType: AuthType; - method: string; - path: string; - body: any; - timestamp: number; -} - export interface ExtendedResponse extends ExpressResponse {} diff --git a/shared/api/common/epochUtils.ts b/shared/api/common/epochUtils.ts index 7b5aa0d92..aa5c4abae 100644 --- a/shared/api/common/epochUtils.ts +++ b/shared/api/common/epochUtils.ts @@ -1,4 +1,4 @@ -/** Converts epoch ms to ClickHouse DateTime string format */ +/** Converts epoch ms to DateTime string format (YYYY-MM-DD HH:MM:SS) */ export const epochToDateTime = (epochMs: number): string => { const date = new Date(epochMs); const year = date.getUTCFullYear(); diff --git a/shared/enums/ErrCode.ts b/shared/enums/ErrCode.ts index 88fd21976..182ea584b 100644 --- a/shared/enums/ErrCode.ts +++ b/shared/enums/ErrCode.ts @@ -159,8 +159,10 @@ export const ErrCode = { // Entities EntityBalanceNotFound: "entity_balance_not_found", - // ClickHouse - ClickHouseDisabled: "clickhouse_disabled", + // Tinybird + TinybirdDisabled: "tinybird_disabled", + /** @deprecated Use TinybirdDisabled instead ā€” kept for frontend compat */ + ClickHouseDisabled: "tinybird_disabled", // Payment method PaymentMethodNotFound: "payment_method_not_found", diff --git a/shared/types/sharedContext.ts b/shared/types/sharedContext.ts index f9aba3599..a8863ad16 100644 --- a/shared/types/sharedContext.ts +++ b/shared/types/sharedContext.ts @@ -13,7 +13,6 @@ export type SharedContext = { logger: AutumnLogger; expand: string[]; // db: DrizzleCli; - // clickhouseClient?: ClickHouseClient; // // Info // id: string; diff --git a/vite/src/views/customers/customer/analytics/hooks/useEventNames.tsx b/vite/src/views/customers/customer/analytics/hooks/useEventNames.tsx index cccca9f08..c9fc86197 100644 --- a/vite/src/views/customers/customer/analytics/hooks/useEventNames.tsx +++ b/vite/src/views/customers/customer/analytics/hooks/useEventNames.tsx @@ -7,11 +7,7 @@ export type EventNameWithCount = { }; export const useEventNames = (limit?: number) => { - const { - data, - isLoading, - error, - } = usePostSWR<{ eventNames: EventNameWithCount[] }>({ + const { data, isLoading, error } = usePostSWR({ method: "get", url: `/query/event_names/list${limit ? `?limit=${limit}` : ""}`, queryKey: ["query-event-names-list", limit], diff --git a/vite/src/views/customers2/components/table/customer-usage-analytics/CustomerUsageAnalyticsTable.tsx b/vite/src/views/customers2/components/table/customer-usage-analytics/CustomerUsageAnalyticsTable.tsx index 2b1296584..9862fde92 100644 --- a/vite/src/views/customers2/components/table/customer-usage-analytics/CustomerUsageAnalyticsTable.tsx +++ b/vite/src/views/customers2/components/table/customer-usage-analytics/CustomerUsageAnalyticsTable.tsx @@ -57,7 +57,7 @@ export function CustomerUsageAnalyticsTable() { // Fetch pre-aggregated timeseries data for the chart ā€” only after raw events // have fully settled (including background revalidations) to avoid firing with // stale cached event names from a previously viewed customer. - // Pass the external customer ID since ClickHouse stores events keyed by that. + // Pass the external customer ID since events are keyed by that. const { timeseriesEvents, isLoading: timeseriesLoading } = useCustomerTimeseriesEvents({ interval, diff --git a/vite/src/views/products/plan/components/edit-plan-details/PlanTypeSection.tsx b/vite/src/views/products/plan/components/edit-plan-details/PlanTypeSection.tsx index 26abdd457..740e52ba3 100644 --- a/vite/src/views/products/plan/components/edit-plan-details/PlanTypeSection.tsx +++ b/vite/src/views/products/plan/components/edit-plan-details/PlanTypeSection.tsx @@ -147,25 +147,6 @@ export const PlanTypeSection = ({ }, ], }); - // handleUpdateBasePrice({ - // amount: "", - // interval: - // ProductItemInterval.Month as unknown as BillingInterval, - // intervalCount: 1, - // }); - // console.log("hey"); - - // console.log(product); - // await setProduct({ - // ...product, - // items: [ - // ...product.items, - // { - // price: 10, - // interval: ProductItemInterval.Month, - // }, - // ], - // }); }} icon={} />