fix: 🐛 only counted row instead of value

This commit is contained in:
amianthus
2025-09-16 14:47:04 +01:00
parent e7f6dffe34
commit 7e201dae41

View File

@@ -1,50 +1,52 @@
import { ClickHouseClient } from "@clickhouse/client"; /** biome-ignore-all lint/complexity/noStaticOnlyClass: wrap it up buddy */
import { ErrCode, FullCustomer } from "@autumn/shared";
import { ExtendedRequest } from "@/utils/models/Request.js"; import { ErrCode, type FullCustomer } from "@autumn/shared";
import RecaseError from "@/utils/errorUtils.js"; import type { ClickHouseClient } from "@clickhouse/client";
import { StatusCodes } from "http-status-codes"; import { StatusCodes } from "http-status-codes";
import RecaseError from "@/utils/errorUtils.js";
import type { ExtendedRequest } from "@/utils/models/Request.js";
import { import {
generateEventCountExpressions, generateEventCountExpressions,
getBillingCycleStartDate, getBillingCycleStartDate,
} from "./analyticsUtils.js"; } from "./analyticsUtils.js";
export class AnalyticsService { export class AnalyticsService {
static clickhouseAvailable = static clickhouseAvailable =
process.env.CLICKHOUSE_URL && process.env.CLICKHOUSE_URL &&
process.env.CLICKHOUSE_USERNAME && process.env.CLICKHOUSE_USERNAME &&
process.env.CLICKHOUSE_PASSWORD; process.env.CLICKHOUSE_PASSWORD;
static handleEarlyExit = () => { static handleEarlyExit = () => {
if (!AnalyticsService.clickhouseAvailable) { if (!AnalyticsService.clickhouseAvailable) {
throw new RecaseError({ throw new RecaseError({
message: "ClickHouse is disabled, cannot fetch events", message: "ClickHouse is disabled, cannot fetch events",
code: ErrCode.ClickHouseDisabled, code: ErrCode.ClickHouseDisabled,
statusCode: StatusCodes.SERVICE_UNAVAILABLE, statusCode: StatusCodes.SERVICE_UNAVAILABLE,
}); });
} }
}; };
static formatJsDateToClickHouseDateTime(date: Date) { static formatJsDateToClickHouseDateTime(date: Date) {
const year = date.getFullYear(); const year = date.getFullYear();
const month = String(date.getMonth() + 1).padStart(2, "0"); const month = String(date.getMonth() + 1).padStart(2, "0");
const day = String(date.getDate()).padStart(2, "0"); const day = String(date.getDate()).padStart(2, "0");
const hours = String(date.getHours()).padStart(2, "0"); const hours = String(date.getHours()).padStart(2, "0");
const minutes = String(date.getMinutes() - 1).padStart(2, "0"); const minutes = String(date.getMinutes() - 1).padStart(2, "0");
const seconds = String(date.getSeconds() - 1).padStart(2, "0"); const seconds = String(date.getSeconds() - 1).padStart(2, "0");
return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`; return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`;
} }
static async getTopEventNames({ static async getTopEventNames({
req, req,
limit = 3, limit = 3,
}: { }: {
req: ExtendedRequest; req: ExtendedRequest;
limit?: number; limit?: number;
}) { }) {
const { clickhouseClient, org, env } = req; const { clickhouseClient, org, env } = req;
const query = ` const query = `
select count(*) as count, event_name select count(*) as count, event_name
from org_events_view(org_id={org_id:String}, org_slug='', env={env:String}) from org_events_view(org_id={org_id:String}, org_slug='', env={env:String})
where timestamp >= NOW() - INTERVAL '1 month' where timestamp >= NOW() - INTERVAL '1 month'
@@ -52,27 +54,27 @@ export class AnalyticsService {
order by count(*) desc order by count(*) desc
limit {limit:UInt32} limit {limit:UInt32}
`; `;
const result = await clickhouseClient.query({ const result = await clickhouseClient.query({
query, query,
query_params: { query_params: {
org_id: org?.id, org_id: org?.id,
env: env, env: env,
limit, limit,
}, },
}); });
const resultJson = await result.json(); const resultJson = await result.json();
return { return {
eventNames: resultJson.data.map((row: any) => row.event_name), eventNames: resultJson.data.map((row: any) => row.event_name),
result: resultJson, result: resultJson,
}; };
} }
static async getTopUser({ req }: { req: ExtendedRequest }) { static async getTopUser({ req }: { req: ExtendedRequest }) {
const { clickhouseClient, org, env, db } = req; const { clickhouseClient, org, env, db } = req;
const query = ` const query = `
SELECT SELECT
c.name c.name
FROM FROM
@@ -113,116 +115,118 @@ WHERE
) )
`; `;
const result = await clickhouseClient.query({ const result = await clickhouseClient.query({
query, query,
query_params: { query_params: {
org_id: org?.id, org_id: org?.id,
env: env, env: env,
}, },
}); });
const resultJson = await result.json(); const resultJson = await result.json();
return (resultJson.data as { name: string; count: number }[])[0]; return (resultJson.data as { name: string; count: number }[])[0];
} }
static async getTotalEvents({ static async getTotalEvents({
req, req,
eventName, eventName,
}: { }: {
req: ExtendedRequest; req: ExtendedRequest;
eventName?: string; eventName?: string;
}) { }) {
const { clickhouseClient, org, env, db } = req; const { clickhouseClient, org, env, db } = req;
const query = ` const query = `
SELECT org_id, env, COUNT(*) AS total_events SELECT SUM(
FROM events CASE
WHERE org_id = {org_id: String} WHEN JSONHas(properties, 'value') THEN toInt64(JSONExtractFloat(properties, 'value'))
AND env = {env: String} WHEN value IS NOT NULL THEN toInt64(value)
${eventName ? `AND event_name = {eventName: String}` : ""} ELSE 1
GROUP BY org_id, env END
LIMIT 1; ) AS total_events
FROM org_events_view(org_id={org_id:String}, org_slug='', env={env:String})
WHERE event_name = {eventName:String}
`; `;
const result = await clickhouseClient.query({ const result = await clickhouseClient.query({
query, query,
query_params: { query_params: {
org_id: org?.id, org_id: org?.id,
env: env, env: env,
eventName: eventName ?? undefined, eventName: eventName ?? undefined,
}, },
}); });
const resultJson = await result.json(); const resultJson = await result.json();
return (resultJson.data as { total_events: number }[])[0].total_events; return (resultJson.data as { total_events: number }[])[0].total_events;
} }
static async getTotalCustomers({ req }: { req: ExtendedRequest }) { static async getTotalCustomers({ req }: { req: ExtendedRequest }) {
const { clickhouseClient, org, env, db } = req; const { clickhouseClient, org, env, db } = req;
const query = `SELECT COUNT(DISTINCT id) AS total_customers const query = `SELECT COUNT(DISTINCT id) AS total_customers
FROM customers FROM customers
WHERE org_id = {org_id:String} WHERE org_id = {org_id:String}
AND env = {env:String};`; AND env = {env:String};`;
const result = await clickhouseClient.query({ const result = await clickhouseClient.query({
query, query,
query_params: { query_params: {
org_id: org?.id, org_id: org?.id,
env: env, env: env,
}, },
}); });
const resultJson = await result.json(); const resultJson = await result.json();
return (resultJson.data as { total_customers: number }[])[0] return (resultJson.data as { total_customers: number }[])[0]
.total_customers; .total_customers;
} }
static async getTimeseriesEvents({ static async getTimeseriesEvents({
req, req,
params, params,
customer, customer,
aggregateAll = false, aggregateAll = false,
}: { }: {
req: ExtendedRequest; req: ExtendedRequest;
params: { params: {
event_names: string[]; event_names: string[];
interval: "24h" | "7d" | "30d" | "90d" | "1bc" | "3bc"; interval: "24h" | "7d" | "30d" | "90d" | "1bc" | "3bc";
customer_id?: string; customer_id?: string;
no_count?: boolean; no_count?: boolean;
}; };
customer?: FullCustomer; customer?: FullCustomer;
aggregateAll?: boolean; aggregateAll?: boolean;
}) { }) {
const { clickhouseClient, org, env, db } = req; const { clickhouseClient, org, env, db } = req;
const intervalType: "24h" | "7d" | "30d" | "90d" | "1bc" | "3bc" = const intervalType: "24h" | "7d" | "30d" | "90d" | "1bc" | "3bc" =
params.interval || "24h"; params.interval || "24h";
const isBillingCycle = intervalType === "1bc" || intervalType === "3bc"; const isBillingCycle = intervalType === "1bc" || intervalType === "3bc";
AnalyticsService.handleEarlyExit(); AnalyticsService.handleEarlyExit();
// Skip billing cycle calculation if aggregating all customers // Skip billing cycle calculation if aggregating all customers
let getBCResults = const getBCResults =
isBillingCycle && !aggregateAll && customer isBillingCycle && !aggregateAll && customer
? ((await getBillingCycleStartDate( ? ((await getBillingCycleStartDate(
env, env,
org?.id, org?.id,
customer, customer,
db, db,
intervalType as "1bc" | "3bc" intervalType as "1bc" | "3bc",
)) as { startDate: string; endDate: string; gap: number } | null) )) as { startDate: string; endDate: string; gap: number } | null)
: null; : null;
const countExpressions = generateEventCountExpressions( const countExpressions = generateEventCountExpressions(
params.event_names, params.event_names,
params.no_count params.no_count,
); );
if (AnalyticsService.clickhouseAvailable) { if (AnalyticsService.clickhouseAvailable) {
const query = ` const query = `
with customer_events as ( with customer_events as (
select * select *
from org_events_view(org_id={org_id:String}, org_slug='', env={env:String}) from org_events_view(org_id={org_id:String}, org_slug='', env={env:String})
@@ -238,7 +242,7 @@ group by dr.period
order by dr.period; order by dr.period;
`; `;
const queryBillingCycle = ` const queryBillingCycle = `
with customer_events as ( with customer_events as (
select * select *
from org_events_view(org_id={org_id:String}, org_slug='', env={env:String}) from org_events_view(org_id={org_id:String}, org_slug='', env={env:String})
@@ -254,118 +258,118 @@ group by dr.period
order by dr.period; order by dr.period;
`; `;
const queryParams = { const queryParams = {
org_id: org?.id, org_id: org?.id,
env: env, env: env,
customer_id: params.customer_id, customer_id: params.customer_id,
days: days:
intervalType === "24h" intervalType === "24h"
? 1 ? 1
: intervalType === "7d" : intervalType === "7d"
? 7 ? 7
: intervalType === "30d" : intervalType === "30d"
? 30 ? 30
: intervalType === "90d" : intervalType === "90d"
? 90 ? 90
: intervalType === "1bc" : intervalType === "1bc"
? (getBCResults?.gap ?? 0) + 1 ? (getBCResults?.gap ?? 0) + 1
: intervalType === "3bc" : intervalType === "3bc"
? (getBCResults?.gap ?? 0) ? (getBCResults?.gap ?? 0)
: 0, : 0,
bin_size: intervalType === "24h" ? "hour" : "day", bin_size: intervalType === "24h" ? "hour" : "day",
end_date: isBillingCycle ? getBCResults?.endDate : undefined, end_date: isBillingCycle ? getBCResults?.endDate : undefined,
}; };
// Use regular query for aggregateAll or when no billing cycle data is available // Use regular query for aggregateAll or when no billing cycle data is available
const queryToUse = const queryToUse =
isBillingCycle && !aggregateAll && getBCResults?.startDate isBillingCycle && !aggregateAll && getBCResults?.startDate
? queryBillingCycle ? queryBillingCycle
: query; : query;
const result = await (clickhouseClient as ClickHouseClient).query({ const result = await (clickhouseClient as ClickHouseClient).query({
query: queryToUse, query: queryToUse,
query_params: queryParams, query_params: queryParams,
format: "JSON", format: "JSON",
clickhouse_settings: { clickhouse_settings: {
output_format_json_quote_decimals: 0, output_format_json_quote_decimals: 0,
output_format_json_quote_64bit_integers: 1, output_format_json_quote_64bit_integers: 1,
output_format_json_quote_64bit_floats: 1, output_format_json_quote_64bit_floats: 1,
}, },
}); });
let resultJson = await result.json(); const resultJson = await result.json();
resultJson.data.forEach((row: any) => { resultJson.data.forEach((row: any) => {
Object.keys(row).forEach((key: string) => { Object.keys(row).forEach((key: string) => {
if (key !== "period") { if (key !== "period") {
row[key] = parseInt(row[key]); row[key] = parseInt(row[key]);
} }
}); });
}); });
return resultJson; return resultJson;
} }
} }
static async getRawEvents({ static async getRawEvents({
req, req,
params, params,
customer, customer,
aggregateAll = false, aggregateAll = false,
}: { }: {
req: ExtendedRequest; req: ExtendedRequest;
params: any; params: any;
customer?: FullCustomer; customer?: FullCustomer;
aggregateAll?: boolean; aggregateAll?: boolean;
}) { }) {
const { clickhouseClient, org, db, env } = req; const { clickhouseClient, org, db, env } = req;
AnalyticsService.handleEarlyExit(); AnalyticsService.handleEarlyExit();
let startDate = new Date(); const startDate = new Date();
const intervalType = params.interval || "day"; const intervalType = params.interval || "day";
const isBillingCycle = intervalType === "1bc" || intervalType === "3bc"; const isBillingCycle = intervalType === "1bc" || intervalType === "3bc";
// Skip billing cycle calculation if aggregating all customers // Skip billing cycle calculation if aggregating all customers
let getBCResults = const getBCResults =
isBillingCycle && !aggregateAll && customer isBillingCycle && !aggregateAll && customer
? ((await getBillingCycleStartDate( ? ((await getBillingCycleStartDate(
env, env,
org?.id, org?.id,
customer, customer,
db, db,
intervalType as "1bc" | "3bc" intervalType as "1bc" | "3bc",
)) as { startDate: string; endDate: string; gap: number } | null) )) as { startDate: string; endDate: string; gap: number } | null)
: null; : null;
switch (intervalType) { switch (intervalType) {
case "24h": case "24h":
startDate.setHours(startDate.getHours() - 24); startDate.setHours(startDate.getHours() - 24);
break; break;
case "7d": case "7d":
startDate.setDate(startDate.getDate() - 7); startDate.setDate(startDate.getDate() - 7);
break; break;
case "30d": case "30d":
startDate.setDate(startDate.getDate() - 30); startDate.setDate(startDate.getDate() - 30);
break; break;
case "90d": case "90d":
startDate.setDate(startDate.getDate() - 90); startDate.setDate(startDate.getDate() - 90);
break; break;
default: default:
startDate.setDate(startDate.getDate() - 24); startDate.setDate(startDate.getDate() - 24);
break; break;
} }
const finalStartDate = const finalStartDate =
isBillingCycle && getBCResults?.startDate isBillingCycle && getBCResults?.startDate
? getBCResults.startDate ? getBCResults.startDate
: AnalyticsService.formatJsDateToClickHouseDateTime(startDate); : AnalyticsService.formatJsDateToClickHouseDateTime(startDate);
const finalEndDate = const finalEndDate =
isBillingCycle && getBCResults?.endDate isBillingCycle && getBCResults?.endDate
? getBCResults.endDate ? getBCResults.endDate
: AnalyticsService.formatJsDateToClickHouseDateTime(new Date()); : AnalyticsService.formatJsDateToClickHouseDateTime(new Date());
const query = ` const query = `
SELECT * SELECT *
FROM org_events_view(org_id={organizationId:String}, org_slug='', env={env:String}) FROM org_events_view(org_id={organizationId:String}, org_slug='', env={env:String})
WHERE timestamp >= toDateTime({startDate:String}) WHERE timestamp >= toDateTime({startDate:String})
@@ -375,49 +379,49 @@ order by dr.period;
limit 10000 limit 10000
`; `;
const filledQuery = query const filledQuery = query
.replace("{organizationId:String}", org?.id ?? "") .replace("{organizationId:String}", org?.id ?? "")
.replace("{customerId:String}", params.customer_id ?? "") .replace("{customerId:String}", params.customer_id ?? "")
.replace("{startDate:String}", finalStartDate) .replace("{startDate:String}", finalStartDate)
.replace("{endDate:String}", finalEndDate) .replace("{endDate:String}", finalEndDate)
.replace("{env:String}", env); .replace("{env:String}", env);
// console.log("filledQuery", filledQuery); // console.log("filledQuery", filledQuery);
const result = await clickhouseClient.query({ const result = await clickhouseClient.query({
query: query, query: query,
query_params: { query_params: {
organizationId: org?.id, organizationId: org?.id,
customerId: params.customer_id, customerId: params.customer_id,
startDate: finalStartDate, startDate: finalStartDate,
endDate: finalEndDate, endDate: finalEndDate,
env: env, env: env,
}, },
}); });
// log the actual query... with params filled in...? // log the actual query... with params filled in...?
// console.log("query", query); // console.log("query", query);
const resultJson = await result.json(); const resultJson = await result.json();
return resultJson; return resultJson;
} }
// private static async getSubscriptionsIfNeeded( // private static async getSubscriptionsIfNeeded(
// customer: FullCustomer, // customer: FullCustomer,
// customerHasSubscriptions: boolean, // customerHasSubscriptions: boolean,
// db: DrizzleCli // db: DrizzleCli
// ): Promise<Subscription[]> { // ): Promise<Subscription[]> {
// if (customerHasSubscriptions) { // if (customerHasSubscriptions) {
// return []; // return [];
// } // }
// return await SubService.getInStripeIds({ // return await SubService.getInStripeIds({
// db, // db,
// ids: // ids:
// customer.customer_products?.flatMap( // customer.customer_products?.flatMap(
// (product: FullCusProduct) => product.subscription_ids ?? [] // (product: FullCusProduct) => product.subscription_ids ?? []
// ) ?? [], // ) ?? [],
// }); // });
// } // }
} }