From 7e201dae41e64c439be7bd2d07b1f0a658bbe15e Mon Sep 17 00:00:00 2001 From: amianthus <49116958+SirTenzin@users.noreply.github.com> Date: Tue, 16 Sep 2025 14:47:04 +0100 Subject: [PATCH] =?UTF-8?q?fix:=20=F0=9F=90=9B=20only=20counted=20row=20in?= =?UTF-8?q?stead=20of=20value?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../internal/analytics/AnalyticsService.ts | 564 +++++++++--------- 1 file changed, 284 insertions(+), 280 deletions(-) diff --git a/server/src/internal/analytics/AnalyticsService.ts b/server/src/internal/analytics/AnalyticsService.ts index a266a0c5c..5e1525cae 100644 --- a/server/src/internal/analytics/AnalyticsService.ts +++ b/server/src/internal/analytics/AnalyticsService.ts @@ -1,50 +1,52 @@ -import { ClickHouseClient } from "@clickhouse/client"; -import { ErrCode, FullCustomer } from "@autumn/shared"; -import { ExtendedRequest } from "@/utils/models/Request.js"; -import RecaseError from "@/utils/errorUtils.js"; +/** biome-ignore-all lint/complexity/noStaticOnlyClass: wrap it up buddy */ + +import { ErrCode, type FullCustomer } from "@autumn/shared"; +import type { ClickHouseClient } from "@clickhouse/client"; import { StatusCodes } from "http-status-codes"; +import RecaseError from "@/utils/errorUtils.js"; +import type { ExtendedRequest } from "@/utils/models/Request.js"; import { generateEventCountExpressions, getBillingCycleStartDate, } from "./analyticsUtils.js"; export class AnalyticsService { - static clickhouseAvailable = - process.env.CLICKHOUSE_URL && - process.env.CLICKHOUSE_USERNAME && - process.env.CLICKHOUSE_PASSWORD; + static clickhouseAvailable = + process.env.CLICKHOUSE_URL && + process.env.CLICKHOUSE_USERNAME && + process.env.CLICKHOUSE_PASSWORD; - static handleEarlyExit = () => { - if (!AnalyticsService.clickhouseAvailable) { - throw new RecaseError({ - message: "ClickHouse is disabled, cannot fetch events", - code: ErrCode.ClickHouseDisabled, - statusCode: StatusCodes.SERVICE_UNAVAILABLE, - }); - } - }; + static handleEarlyExit = () => { + if (!AnalyticsService.clickhouseAvailable) { + throw new RecaseError({ + message: "ClickHouse is disabled, cannot fetch events", + code: ErrCode.ClickHouseDisabled, + statusCode: StatusCodes.SERVICE_UNAVAILABLE, + }); + } + }; - static formatJsDateToClickHouseDateTime(date: Date) { - const year = date.getFullYear(); - const month = String(date.getMonth() + 1).padStart(2, "0"); - const day = String(date.getDate()).padStart(2, "0"); - const hours = String(date.getHours()).padStart(2, "0"); - const minutes = String(date.getMinutes() - 1).padStart(2, "0"); - const seconds = String(date.getSeconds() - 1).padStart(2, "0"); + static formatJsDateToClickHouseDateTime(date: Date) { + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, "0"); + const day = String(date.getDate()).padStart(2, "0"); + const hours = String(date.getHours()).padStart(2, "0"); + const minutes = String(date.getMinutes() - 1).padStart(2, "0"); + const seconds = String(date.getSeconds() - 1).padStart(2, "0"); - return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`; - } + return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`; + } - static async getTopEventNames({ - req, - limit = 3, - }: { - req: ExtendedRequest; - limit?: number; - }) { - const { clickhouseClient, org, env } = req; + static async getTopEventNames({ + req, + limit = 3, + }: { + req: ExtendedRequest; + limit?: number; + }) { + const { clickhouseClient, org, env } = req; - const query = ` + const query = ` select count(*) as count, event_name from org_events_view(org_id={org_id:String}, org_slug='', env={env:String}) where timestamp >= NOW() - INTERVAL '1 month' @@ -52,27 +54,27 @@ export class AnalyticsService { order by count(*) desc limit {limit:UInt32} `; - const result = await clickhouseClient.query({ - query, - query_params: { - org_id: org?.id, - env: env, - limit, - }, - }); + const result = await clickhouseClient.query({ + query, + query_params: { + org_id: org?.id, + env: env, + limit, + }, + }); - const resultJson = await result.json(); + const resultJson = await result.json(); - return { - eventNames: resultJson.data.map((row: any) => row.event_name), - result: resultJson, - }; - } + return { + eventNames: resultJson.data.map((row: any) => row.event_name), + result: resultJson, + }; + } - static async getTopUser({ req }: { req: ExtendedRequest }) { - const { clickhouseClient, org, env, db } = req; + static async getTopUser({ req }: { req: ExtendedRequest }) { + const { clickhouseClient, org, env, db } = req; - const query = ` + const query = ` SELECT c.name FROM @@ -113,116 +115,118 @@ WHERE ) `; - const result = await clickhouseClient.query({ - query, - query_params: { - org_id: org?.id, - env: env, - }, - }); + const result = await clickhouseClient.query({ + query, + query_params: { + org_id: org?.id, + env: env, + }, + }); - const resultJson = await result.json(); + const resultJson = await result.json(); - return (resultJson.data as { name: string; count: number }[])[0]; - } + return (resultJson.data as { name: string; count: number }[])[0]; + } - static async getTotalEvents({ - req, - eventName, - }: { - req: ExtendedRequest; - eventName?: string; - }) { - const { clickhouseClient, org, env, db } = req; + static async getTotalEvents({ + req, + eventName, + }: { + req: ExtendedRequest; + eventName?: string; + }) { + const { clickhouseClient, org, env, db } = req; - const query = ` -SELECT org_id, env, COUNT(*) AS total_events -FROM events -WHERE org_id = {org_id: String} - AND env = {env: String} - ${eventName ? `AND event_name = {eventName: String}` : ""} -GROUP BY org_id, env -LIMIT 1; + const query = ` +SELECT SUM( + CASE + WHEN JSONHas(properties, 'value') THEN toInt64(JSONExtractFloat(properties, 'value')) + WHEN value IS NOT NULL THEN toInt64(value) + ELSE 1 + END +) AS total_events +FROM org_events_view(org_id={org_id:String}, org_slug='', env={env:String}) +WHERE event_name = {eventName:String} `; - const result = await clickhouseClient.query({ - query, - query_params: { - org_id: org?.id, - env: env, - eventName: eventName ?? undefined, - }, - }); + const result = await clickhouseClient.query({ + query, + query_params: { + org_id: org?.id, + env: env, + eventName: eventName ?? undefined, + }, + }); - const resultJson = await result.json(); + const resultJson = await result.json(); - return (resultJson.data as { total_events: number }[])[0].total_events; - } + return (resultJson.data as { total_events: number }[])[0].total_events; + } - static async getTotalCustomers({ req }: { req: ExtendedRequest }) { - const { clickhouseClient, org, env, db } = req; - const query = `SELECT COUNT(DISTINCT id) AS total_customers + static async getTotalCustomers({ req }: { req: ExtendedRequest }) { + const { clickhouseClient, org, env, db } = req; + const query = `SELECT COUNT(DISTINCT id) AS total_customers FROM customers WHERE org_id = {org_id:String} AND env = {env:String};`; - const result = await clickhouseClient.query({ - query, - query_params: { - org_id: org?.id, - env: env, - }, - }); + const result = await clickhouseClient.query({ + query, + query_params: { + org_id: org?.id, + env: env, + }, + }); - const resultJson = await result.json(); + const resultJson = await result.json(); - return (resultJson.data as { total_customers: number }[])[0] - .total_customers; - } + return (resultJson.data as { total_customers: number }[])[0] + .total_customers; + } - static async getTimeseriesEvents({ - req, - params, - customer, - aggregateAll = false, - }: { - req: ExtendedRequest; - params: { - event_names: string[]; - interval: "24h" | "7d" | "30d" | "90d" | "1bc" | "3bc"; - customer_id?: string; - no_count?: boolean; - }; - customer?: FullCustomer; - aggregateAll?: boolean; - }) { - const { clickhouseClient, org, env, db } = req; + static async getTimeseriesEvents({ + req, + params, + customer, + aggregateAll = false, + }: { + req: ExtendedRequest; + params: { + event_names: string[]; + interval: "24h" | "7d" | "30d" | "90d" | "1bc" | "3bc"; + customer_id?: string; + no_count?: boolean; + }; + customer?: FullCustomer; + aggregateAll?: boolean; + }) { + const { clickhouseClient, org, env, db } = req; - const intervalType: "24h" | "7d" | "30d" | "90d" | "1bc" | "3bc" = - params.interval || "24h"; + const intervalType: "24h" | "7d" | "30d" | "90d" | "1bc" | "3bc" = + params.interval || "24h"; - const isBillingCycle = intervalType === "1bc" || intervalType === "3bc"; - AnalyticsService.handleEarlyExit(); + const isBillingCycle = intervalType === "1bc" || intervalType === "3bc"; + AnalyticsService.handleEarlyExit(); - // Skip billing cycle calculation if aggregating all customers - let getBCResults = - isBillingCycle && !aggregateAll && customer - ? ((await getBillingCycleStartDate( - env, - org?.id, - customer, - db, - intervalType as "1bc" | "3bc" - )) as { startDate: string; endDate: string; gap: number } | null) - : null; + // Skip billing cycle calculation if aggregating all customers + const getBCResults = + isBillingCycle && !aggregateAll && customer + ? ((await getBillingCycleStartDate( + env, + org?.id, + customer, + db, + intervalType as "1bc" | "3bc", + )) as { startDate: string; endDate: string; gap: number } | null) + : null; - const countExpressions = generateEventCountExpressions( - params.event_names, - params.no_count - ); + const countExpressions = generateEventCountExpressions( + params.event_names, + params.no_count, + ); - if (AnalyticsService.clickhouseAvailable) { - const query = ` + if (AnalyticsService.clickhouseAvailable) { + const query = ` with customer_events as ( select * from org_events_view(org_id={org_id:String}, org_slug='', env={env:String}) @@ -238,7 +242,7 @@ group by dr.period order by dr.period; `; - const queryBillingCycle = ` + const queryBillingCycle = ` with customer_events as ( select * from org_events_view(org_id={org_id:String}, org_slug='', env={env:String}) @@ -254,118 +258,118 @@ group by dr.period order by dr.period; `; - const queryParams = { - org_id: org?.id, - env: env, - customer_id: params.customer_id, - days: - intervalType === "24h" - ? 1 - : intervalType === "7d" - ? 7 - : intervalType === "30d" - ? 30 - : intervalType === "90d" - ? 90 - : intervalType === "1bc" - ? (getBCResults?.gap ?? 0) + 1 - : intervalType === "3bc" - ? (getBCResults?.gap ?? 0) - : 0, - bin_size: intervalType === "24h" ? "hour" : "day", - end_date: isBillingCycle ? getBCResults?.endDate : undefined, - }; + const queryParams = { + org_id: org?.id, + env: env, + customer_id: params.customer_id, + days: + intervalType === "24h" + ? 1 + : intervalType === "7d" + ? 7 + : intervalType === "30d" + ? 30 + : intervalType === "90d" + ? 90 + : intervalType === "1bc" + ? (getBCResults?.gap ?? 0) + 1 + : intervalType === "3bc" + ? (getBCResults?.gap ?? 0) + : 0, + bin_size: intervalType === "24h" ? "hour" : "day", + end_date: isBillingCycle ? getBCResults?.endDate : undefined, + }; - // Use regular query for aggregateAll or when no billing cycle data is available - const queryToUse = - isBillingCycle && !aggregateAll && getBCResults?.startDate - ? queryBillingCycle - : query; + // Use regular query for aggregateAll or when no billing cycle data is available + const queryToUse = + isBillingCycle && !aggregateAll && getBCResults?.startDate + ? queryBillingCycle + : query; - const result = await (clickhouseClient as ClickHouseClient).query({ - query: queryToUse, - query_params: queryParams, - format: "JSON", - clickhouse_settings: { - output_format_json_quote_decimals: 0, - output_format_json_quote_64bit_integers: 1, - output_format_json_quote_64bit_floats: 1, - }, - }); + const result = await (clickhouseClient as ClickHouseClient).query({ + query: queryToUse, + query_params: queryParams, + format: "JSON", + clickhouse_settings: { + output_format_json_quote_decimals: 0, + output_format_json_quote_64bit_integers: 1, + output_format_json_quote_64bit_floats: 1, + }, + }); - let resultJson = await result.json(); + const resultJson = await result.json(); - resultJson.data.forEach((row: any) => { - Object.keys(row).forEach((key: string) => { - if (key !== "period") { - row[key] = parseInt(row[key]); - } - }); - }); + resultJson.data.forEach((row: any) => { + Object.keys(row).forEach((key: string) => { + if (key !== "period") { + row[key] = parseInt(row[key]); + } + }); + }); - return resultJson; - } - } + return resultJson; + } + } - static async getRawEvents({ - req, - params, - customer, - aggregateAll = false, - }: { - req: ExtendedRequest; - params: any; - customer?: FullCustomer; - aggregateAll?: boolean; - }) { - const { clickhouseClient, org, db, env } = req; + static async getRawEvents({ + req, + params, + customer, + aggregateAll = false, + }: { + req: ExtendedRequest; + params: any; + customer?: FullCustomer; + aggregateAll?: boolean; + }) { + const { clickhouseClient, org, db, env } = req; - AnalyticsService.handleEarlyExit(); + AnalyticsService.handleEarlyExit(); - let startDate = new Date(); - const intervalType = params.interval || "day"; - const isBillingCycle = intervalType === "1bc" || intervalType === "3bc"; + const startDate = new Date(); + const intervalType = params.interval || "day"; + const isBillingCycle = intervalType === "1bc" || intervalType === "3bc"; - // Skip billing cycle calculation if aggregating all customers - let getBCResults = - isBillingCycle && !aggregateAll && customer - ? ((await getBillingCycleStartDate( - env, - org?.id, - customer, - db, - intervalType as "1bc" | "3bc" - )) as { startDate: string; endDate: string; gap: number } | null) - : null; + // Skip billing cycle calculation if aggregating all customers + const getBCResults = + isBillingCycle && !aggregateAll && customer + ? ((await getBillingCycleStartDate( + env, + org?.id, + customer, + db, + intervalType as "1bc" | "3bc", + )) as { startDate: string; endDate: string; gap: number } | null) + : null; - switch (intervalType) { - case "24h": - startDate.setHours(startDate.getHours() - 24); - break; - case "7d": - startDate.setDate(startDate.getDate() - 7); - break; - case "30d": - startDate.setDate(startDate.getDate() - 30); - break; - case "90d": - startDate.setDate(startDate.getDate() - 90); - break; - default: - startDate.setDate(startDate.getDate() - 24); - break; - } + switch (intervalType) { + case "24h": + startDate.setHours(startDate.getHours() - 24); + break; + case "7d": + startDate.setDate(startDate.getDate() - 7); + break; + case "30d": + startDate.setDate(startDate.getDate() - 30); + break; + case "90d": + startDate.setDate(startDate.getDate() - 90); + break; + default: + startDate.setDate(startDate.getDate() - 24); + break; + } - const finalStartDate = - isBillingCycle && getBCResults?.startDate - ? getBCResults.startDate - : AnalyticsService.formatJsDateToClickHouseDateTime(startDate); - const finalEndDate = - isBillingCycle && getBCResults?.endDate - ? getBCResults.endDate - : AnalyticsService.formatJsDateToClickHouseDateTime(new Date()); + const finalStartDate = + isBillingCycle && getBCResults?.startDate + ? getBCResults.startDate + : AnalyticsService.formatJsDateToClickHouseDateTime(startDate); + const finalEndDate = + isBillingCycle && getBCResults?.endDate + ? getBCResults.endDate + : AnalyticsService.formatJsDateToClickHouseDateTime(new Date()); - const query = ` + const query = ` SELECT * FROM org_events_view(org_id={organizationId:String}, org_slug='', env={env:String}) WHERE timestamp >= toDateTime({startDate:String}) @@ -375,49 +379,49 @@ order by dr.period; limit 10000 `; - const filledQuery = query - .replace("{organizationId:String}", org?.id ?? "") - .replace("{customerId:String}", params.customer_id ?? "") - .replace("{startDate:String}", finalStartDate) - .replace("{endDate:String}", finalEndDate) - .replace("{env:String}", env); + const filledQuery = query + .replace("{organizationId:String}", org?.id ?? "") + .replace("{customerId:String}", params.customer_id ?? "") + .replace("{startDate:String}", finalStartDate) + .replace("{endDate:String}", finalEndDate) + .replace("{env:String}", env); - // console.log("filledQuery", filledQuery); + // console.log("filledQuery", filledQuery); - const result = await clickhouseClient.query({ - query: query, - query_params: { - organizationId: org?.id, - customerId: params.customer_id, - startDate: finalStartDate, - endDate: finalEndDate, - env: env, - }, - }); + const result = await clickhouseClient.query({ + query: query, + query_params: { + organizationId: org?.id, + customerId: params.customer_id, + startDate: finalStartDate, + endDate: finalEndDate, + env: env, + }, + }); - // log the actual query... with params filled in...? - // console.log("query", query); + // log the actual query... with params filled in...? + // console.log("query", query); - const resultJson = await result.json(); + const resultJson = await result.json(); - return resultJson; - } + return resultJson; + } - // private static async getSubscriptionsIfNeeded( - // customer: FullCustomer, - // customerHasSubscriptions: boolean, - // db: DrizzleCli - // ): Promise { - // if (customerHasSubscriptions) { - // return []; - // } + // private static async getSubscriptionsIfNeeded( + // customer: FullCustomer, + // customerHasSubscriptions: boolean, + // db: DrizzleCli + // ): Promise { + // if (customerHasSubscriptions) { + // return []; + // } - // return await SubService.getInStripeIds({ - // db, - // ids: - // customer.customer_products?.flatMap( - // (product: FullCusProduct) => product.subscription_ids ?? [] - // ) ?? [], - // }); - // } + // return await SubService.getInStripeIds({ + // db, + // ids: + // customer.customer_products?.flatMap( + // (product: FullCusProduct) => product.subscription_ids ?? [] + // ) ?? [], + // }); + // } }