fix: 🐛 only counted row instead of value

This commit is contained in:
amianthus
2025-09-16 14:47:04 +01:00
parent e7f6dffe34
commit 7e201dae41

View File

@@ -1,50 +1,52 @@
import { ClickHouseClient } from "@clickhouse/client";
import { ErrCode, FullCustomer } from "@autumn/shared";
import { ExtendedRequest } from "@/utils/models/Request.js";
import RecaseError from "@/utils/errorUtils.js";
/** biome-ignore-all lint/complexity/noStaticOnlyClass: wrap it up buddy */
import { ErrCode, type FullCustomer } from "@autumn/shared";
import type { ClickHouseClient } from "@clickhouse/client";
import { StatusCodes } from "http-status-codes";
import RecaseError from "@/utils/errorUtils.js";
import type { ExtendedRequest } from "@/utils/models/Request.js";
import {
generateEventCountExpressions,
getBillingCycleStartDate,
} from "./analyticsUtils.js";
export class AnalyticsService {
static clickhouseAvailable =
process.env.CLICKHOUSE_URL &&
process.env.CLICKHOUSE_USERNAME &&
process.env.CLICKHOUSE_PASSWORD;
static clickhouseAvailable =
process.env.CLICKHOUSE_URL &&
process.env.CLICKHOUSE_USERNAME &&
process.env.CLICKHOUSE_PASSWORD;
static handleEarlyExit = () => {
if (!AnalyticsService.clickhouseAvailable) {
throw new RecaseError({
message: "ClickHouse is disabled, cannot fetch events",
code: ErrCode.ClickHouseDisabled,
statusCode: StatusCodes.SERVICE_UNAVAILABLE,
});
}
};
static handleEarlyExit = () => {
if (!AnalyticsService.clickhouseAvailable) {
throw new RecaseError({
message: "ClickHouse is disabled, cannot fetch events",
code: ErrCode.ClickHouseDisabled,
statusCode: StatusCodes.SERVICE_UNAVAILABLE,
});
}
};
static formatJsDateToClickHouseDateTime(date: Date) {
const year = date.getFullYear();
const month = String(date.getMonth() + 1).padStart(2, "0");
const day = String(date.getDate()).padStart(2, "0");
const hours = String(date.getHours()).padStart(2, "0");
const minutes = String(date.getMinutes() - 1).padStart(2, "0");
const seconds = String(date.getSeconds() - 1).padStart(2, "0");
static formatJsDateToClickHouseDateTime(date: Date) {
const year = date.getFullYear();
const month = String(date.getMonth() + 1).padStart(2, "0");
const day = String(date.getDate()).padStart(2, "0");
const hours = String(date.getHours()).padStart(2, "0");
const minutes = String(date.getMinutes() - 1).padStart(2, "0");
const seconds = String(date.getSeconds() - 1).padStart(2, "0");
return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`;
}
return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`;
}
static async getTopEventNames({
req,
limit = 3,
}: {
req: ExtendedRequest;
limit?: number;
}) {
const { clickhouseClient, org, env } = req;
static async getTopEventNames({
req,
limit = 3,
}: {
req: ExtendedRequest;
limit?: number;
}) {
const { clickhouseClient, org, env } = req;
const query = `
const query = `
select count(*) as count, event_name
from org_events_view(org_id={org_id:String}, org_slug='', env={env:String})
where timestamp >= NOW() - INTERVAL '1 month'
@@ -52,27 +54,27 @@ export class AnalyticsService {
order by count(*) desc
limit {limit:UInt32}
`;
const result = await clickhouseClient.query({
query,
query_params: {
org_id: org?.id,
env: env,
limit,
},
});
const result = await clickhouseClient.query({
query,
query_params: {
org_id: org?.id,
env: env,
limit,
},
});
const resultJson = await result.json();
const resultJson = await result.json();
return {
eventNames: resultJson.data.map((row: any) => row.event_name),
result: resultJson,
};
}
return {
eventNames: resultJson.data.map((row: any) => row.event_name),
result: resultJson,
};
}
static async getTopUser({ req }: { req: ExtendedRequest }) {
const { clickhouseClient, org, env, db } = req;
static async getTopUser({ req }: { req: ExtendedRequest }) {
const { clickhouseClient, org, env, db } = req;
const query = `
const query = `
SELECT
c.name
FROM
@@ -113,116 +115,118 @@ WHERE
)
`;
const result = await clickhouseClient.query({
query,
query_params: {
org_id: org?.id,
env: env,
},
});
const result = await clickhouseClient.query({
query,
query_params: {
org_id: org?.id,
env: env,
},
});
const resultJson = await result.json();
const resultJson = await result.json();
return (resultJson.data as { name: string; count: number }[])[0];
}
return (resultJson.data as { name: string; count: number }[])[0];
}
static async getTotalEvents({
req,
eventName,
}: {
req: ExtendedRequest;
eventName?: string;
}) {
const { clickhouseClient, org, env, db } = req;
static async getTotalEvents({
req,
eventName,
}: {
req: ExtendedRequest;
eventName?: string;
}) {
const { clickhouseClient, org, env, db } = req;
const query = `
SELECT org_id, env, COUNT(*) AS total_events
FROM events
WHERE org_id = {org_id: String}
AND env = {env: String}
${eventName ? `AND event_name = {eventName: String}` : ""}
GROUP BY org_id, env
LIMIT 1;
const query = `
SELECT SUM(
CASE
WHEN JSONHas(properties, 'value') THEN toInt64(JSONExtractFloat(properties, 'value'))
WHEN value IS NOT NULL THEN toInt64(value)
ELSE 1
END
) AS total_events
FROM org_events_view(org_id={org_id:String}, org_slug='', env={env:String})
WHERE event_name = {eventName:String}
`;
const result = await clickhouseClient.query({
query,
query_params: {
org_id: org?.id,
env: env,
eventName: eventName ?? undefined,
},
});
const result = await clickhouseClient.query({
query,
query_params: {
org_id: org?.id,
env: env,
eventName: eventName ?? undefined,
},
});
const resultJson = await result.json();
const resultJson = await result.json();
return (resultJson.data as { total_events: number }[])[0].total_events;
}
return (resultJson.data as { total_events: number }[])[0].total_events;
}
static async getTotalCustomers({ req }: { req: ExtendedRequest }) {
const { clickhouseClient, org, env, db } = req;
const query = `SELECT COUNT(DISTINCT id) AS total_customers
static async getTotalCustomers({ req }: { req: ExtendedRequest }) {
const { clickhouseClient, org, env, db } = req;
const query = `SELECT COUNT(DISTINCT id) AS total_customers
FROM customers
WHERE org_id = {org_id:String}
AND env = {env:String};`;
const result = await clickhouseClient.query({
query,
query_params: {
org_id: org?.id,
env: env,
},
});
const result = await clickhouseClient.query({
query,
query_params: {
org_id: org?.id,
env: env,
},
});
const resultJson = await result.json();
const resultJson = await result.json();
return (resultJson.data as { total_customers: number }[])[0]
.total_customers;
}
return (resultJson.data as { total_customers: number }[])[0]
.total_customers;
}
static async getTimeseriesEvents({
req,
params,
customer,
aggregateAll = false,
}: {
req: ExtendedRequest;
params: {
event_names: string[];
interval: "24h" | "7d" | "30d" | "90d" | "1bc" | "3bc";
customer_id?: string;
no_count?: boolean;
};
customer?: FullCustomer;
aggregateAll?: boolean;
}) {
const { clickhouseClient, org, env, db } = req;
static async getTimeseriesEvents({
req,
params,
customer,
aggregateAll = false,
}: {
req: ExtendedRequest;
params: {
event_names: string[];
interval: "24h" | "7d" | "30d" | "90d" | "1bc" | "3bc";
customer_id?: string;
no_count?: boolean;
};
customer?: FullCustomer;
aggregateAll?: boolean;
}) {
const { clickhouseClient, org, env, db } = req;
const intervalType: "24h" | "7d" | "30d" | "90d" | "1bc" | "3bc" =
params.interval || "24h";
const intervalType: "24h" | "7d" | "30d" | "90d" | "1bc" | "3bc" =
params.interval || "24h";
const isBillingCycle = intervalType === "1bc" || intervalType === "3bc";
AnalyticsService.handleEarlyExit();
const isBillingCycle = intervalType === "1bc" || intervalType === "3bc";
AnalyticsService.handleEarlyExit();
// Skip billing cycle calculation if aggregating all customers
let getBCResults =
isBillingCycle && !aggregateAll && customer
? ((await getBillingCycleStartDate(
env,
org?.id,
customer,
db,
intervalType as "1bc" | "3bc"
)) as { startDate: string; endDate: string; gap: number } | null)
: null;
// Skip billing cycle calculation if aggregating all customers
const getBCResults =
isBillingCycle && !aggregateAll && customer
? ((await getBillingCycleStartDate(
env,
org?.id,
customer,
db,
intervalType as "1bc" | "3bc",
)) as { startDate: string; endDate: string; gap: number } | null)
: null;
const countExpressions = generateEventCountExpressions(
params.event_names,
params.no_count
);
const countExpressions = generateEventCountExpressions(
params.event_names,
params.no_count,
);
if (AnalyticsService.clickhouseAvailable) {
const query = `
if (AnalyticsService.clickhouseAvailable) {
const query = `
with customer_events as (
select *
from org_events_view(org_id={org_id:String}, org_slug='', env={env:String})
@@ -238,7 +242,7 @@ group by dr.period
order by dr.period;
`;
const queryBillingCycle = `
const queryBillingCycle = `
with customer_events as (
select *
from org_events_view(org_id={org_id:String}, org_slug='', env={env:String})
@@ -254,118 +258,118 @@ group by dr.period
order by dr.period;
`;
const queryParams = {
org_id: org?.id,
env: env,
customer_id: params.customer_id,
days:
intervalType === "24h"
? 1
: intervalType === "7d"
? 7
: intervalType === "30d"
? 30
: intervalType === "90d"
? 90
: intervalType === "1bc"
? (getBCResults?.gap ?? 0) + 1
: intervalType === "3bc"
? (getBCResults?.gap ?? 0)
: 0,
bin_size: intervalType === "24h" ? "hour" : "day",
end_date: isBillingCycle ? getBCResults?.endDate : undefined,
};
const queryParams = {
org_id: org?.id,
env: env,
customer_id: params.customer_id,
days:
intervalType === "24h"
? 1
: intervalType === "7d"
? 7
: intervalType === "30d"
? 30
: intervalType === "90d"
? 90
: intervalType === "1bc"
? (getBCResults?.gap ?? 0) + 1
: intervalType === "3bc"
? (getBCResults?.gap ?? 0)
: 0,
bin_size: intervalType === "24h" ? "hour" : "day",
end_date: isBillingCycle ? getBCResults?.endDate : undefined,
};
// Use regular query for aggregateAll or when no billing cycle data is available
const queryToUse =
isBillingCycle && !aggregateAll && getBCResults?.startDate
? queryBillingCycle
: query;
// Use regular query for aggregateAll or when no billing cycle data is available
const queryToUse =
isBillingCycle && !aggregateAll && getBCResults?.startDate
? queryBillingCycle
: query;
const result = await (clickhouseClient as ClickHouseClient).query({
query: queryToUse,
query_params: queryParams,
format: "JSON",
clickhouse_settings: {
output_format_json_quote_decimals: 0,
output_format_json_quote_64bit_integers: 1,
output_format_json_quote_64bit_floats: 1,
},
});
const result = await (clickhouseClient as ClickHouseClient).query({
query: queryToUse,
query_params: queryParams,
format: "JSON",
clickhouse_settings: {
output_format_json_quote_decimals: 0,
output_format_json_quote_64bit_integers: 1,
output_format_json_quote_64bit_floats: 1,
},
});
let resultJson = await result.json();
const resultJson = await result.json();
resultJson.data.forEach((row: any) => {
Object.keys(row).forEach((key: string) => {
if (key !== "period") {
row[key] = parseInt(row[key]);
}
});
});
resultJson.data.forEach((row: any) => {
Object.keys(row).forEach((key: string) => {
if (key !== "period") {
row[key] = parseInt(row[key]);
}
});
});
return resultJson;
}
}
return resultJson;
}
}
static async getRawEvents({
req,
params,
customer,
aggregateAll = false,
}: {
req: ExtendedRequest;
params: any;
customer?: FullCustomer;
aggregateAll?: boolean;
}) {
const { clickhouseClient, org, db, env } = req;
static async getRawEvents({
req,
params,
customer,
aggregateAll = false,
}: {
req: ExtendedRequest;
params: any;
customer?: FullCustomer;
aggregateAll?: boolean;
}) {
const { clickhouseClient, org, db, env } = req;
AnalyticsService.handleEarlyExit();
AnalyticsService.handleEarlyExit();
let startDate = new Date();
const intervalType = params.interval || "day";
const isBillingCycle = intervalType === "1bc" || intervalType === "3bc";
const startDate = new Date();
const intervalType = params.interval || "day";
const isBillingCycle = intervalType === "1bc" || intervalType === "3bc";
// Skip billing cycle calculation if aggregating all customers
let getBCResults =
isBillingCycle && !aggregateAll && customer
? ((await getBillingCycleStartDate(
env,
org?.id,
customer,
db,
intervalType as "1bc" | "3bc"
)) as { startDate: string; endDate: string; gap: number } | null)
: null;
// Skip billing cycle calculation if aggregating all customers
const getBCResults =
isBillingCycle && !aggregateAll && customer
? ((await getBillingCycleStartDate(
env,
org?.id,
customer,
db,
intervalType as "1bc" | "3bc",
)) as { startDate: string; endDate: string; gap: number } | null)
: null;
switch (intervalType) {
case "24h":
startDate.setHours(startDate.getHours() - 24);
break;
case "7d":
startDate.setDate(startDate.getDate() - 7);
break;
case "30d":
startDate.setDate(startDate.getDate() - 30);
break;
case "90d":
startDate.setDate(startDate.getDate() - 90);
break;
default:
startDate.setDate(startDate.getDate() - 24);
break;
}
switch (intervalType) {
case "24h":
startDate.setHours(startDate.getHours() - 24);
break;
case "7d":
startDate.setDate(startDate.getDate() - 7);
break;
case "30d":
startDate.setDate(startDate.getDate() - 30);
break;
case "90d":
startDate.setDate(startDate.getDate() - 90);
break;
default:
startDate.setDate(startDate.getDate() - 24);
break;
}
const finalStartDate =
isBillingCycle && getBCResults?.startDate
? getBCResults.startDate
: AnalyticsService.formatJsDateToClickHouseDateTime(startDate);
const finalEndDate =
isBillingCycle && getBCResults?.endDate
? getBCResults.endDate
: AnalyticsService.formatJsDateToClickHouseDateTime(new Date());
const finalStartDate =
isBillingCycle && getBCResults?.startDate
? getBCResults.startDate
: AnalyticsService.formatJsDateToClickHouseDateTime(startDate);
const finalEndDate =
isBillingCycle && getBCResults?.endDate
? getBCResults.endDate
: AnalyticsService.formatJsDateToClickHouseDateTime(new Date());
const query = `
const query = `
SELECT *
FROM org_events_view(org_id={organizationId:String}, org_slug='', env={env:String})
WHERE timestamp >= toDateTime({startDate:String})
@@ -375,49 +379,49 @@ order by dr.period;
limit 10000
`;
const filledQuery = query
.replace("{organizationId:String}", org?.id ?? "")
.replace("{customerId:String}", params.customer_id ?? "")
.replace("{startDate:String}", finalStartDate)
.replace("{endDate:String}", finalEndDate)
.replace("{env:String}", env);
const filledQuery = query
.replace("{organizationId:String}", org?.id ?? "")
.replace("{customerId:String}", params.customer_id ?? "")
.replace("{startDate:String}", finalStartDate)
.replace("{endDate:String}", finalEndDate)
.replace("{env:String}", env);
// console.log("filledQuery", filledQuery);
// console.log("filledQuery", filledQuery);
const result = await clickhouseClient.query({
query: query,
query_params: {
organizationId: org?.id,
customerId: params.customer_id,
startDate: finalStartDate,
endDate: finalEndDate,
env: env,
},
});
const result = await clickhouseClient.query({
query: query,
query_params: {
organizationId: org?.id,
customerId: params.customer_id,
startDate: finalStartDate,
endDate: finalEndDate,
env: env,
},
});
// log the actual query... with params filled in...?
// console.log("query", query);
// log the actual query... with params filled in...?
// console.log("query", query);
const resultJson = await result.json();
const resultJson = await result.json();
return resultJson;
}
return resultJson;
}
// private static async getSubscriptionsIfNeeded(
// customer: FullCustomer,
// customerHasSubscriptions: boolean,
// db: DrizzleCli
// ): Promise<Subscription[]> {
// if (customerHasSubscriptions) {
// return [];
// }
// private static async getSubscriptionsIfNeeded(
// customer: FullCustomer,
// customerHasSubscriptions: boolean,
// db: DrizzleCli
// ): Promise<Subscription[]> {
// if (customerHasSubscriptions) {
// return [];
// }
// return await SubService.getInStripeIds({
// db,
// ids:
// customer.customer_products?.flatMap(
// (product: FullCusProduct) => product.subscription_ids ?? []
// ) ?? [],
// });
// }
// return await SubService.getInStripeIds({
// db,
// ids:
// customer.customer_products?.flatMap(
// (product: FullCusProduct) => product.subscription_ids ?? []
// ) ?? [],
// });
// }
}