diff --git a/server/tinybird/connections/firecrawl_backfill_apikeyid.connection b/server/tinybird/connections/firecrawl_backfill_apikeyid.connection new file mode 100644 index 000000000..2e0c6bdf3 --- /dev/null +++ b/server/tinybird/connections/firecrawl_backfill_apikeyid.connection @@ -0,0 +1,4 @@ +TYPE s3 +S3_ARN {{ tb_secret("s3_role_arn_firecrawl_backfill_apikeyid_75f684b3") }} +S3_REGION us-west-2 +# Learn more at https://www.tinybird.co/docs/forward/get-data-in/connectors/s3#s3-connection-settings diff --git a/server/tinybird/datasources/events.datasource b/server/tinybird/datasources/events.datasource index ae57d24ac..ddd9ed2fa 100644 --- a/server/tinybird/datasources/events.datasource +++ b/server/tinybird/datasources/events.datasource @@ -21,3 +21,7 @@ SCHEMA > ENGINE "MergeTree" ENGINE_PARTITION_KEY "toYYYYMM(timestamp)" ENGINE_SORTING_KEY "org_id, env, customer_id, event_name, timestamp" + +IMPORT_CONNECTION_NAME firecrawl_backfill_apikeyid +IMPORT_BUCKET_URI s3://firecrawl-emergency-backfill-apikeyid/v2/**/*.ndjson +IMPORT_SCHEDULE @auto \ No newline at end of file diff --git a/server/tinybird/materializations/events_hourly_no_properties_mv.datasource b/server/tinybird/materializations/events_hourly_no_properties_mv.datasource new file mode 100644 index 000000000..52c397b3e --- /dev/null +++ b/server/tinybird/materializations/events_hourly_no_properties_mv.datasource @@ -0,0 +1,20 @@ +DESCRIPTION > + Hourly aggregates WITHOUT properties for fast aggregation queries. + Used by aggregate_simple when no property filters are needed. + Drops the properties column from the GROUP BY so rows with different + properties but the same (org, env, customer, event, entity, hour) + are collapsed into a single row. + +SCHEMA > + `org_id` String, + `env` String, + `customer_id` String, + `event_name` String, + `entity_id` String DEFAULT '', + `hour` DateTime, + `total_value` Float64, + `event_count` UInt64 + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(hour)" +ENGINE_SORTING_KEY "org_id, env, hour, customer_id, entity_id, event_name" \ No newline at end of file diff --git a/server/tinybird/materializations/events_hourly_no_properties_mv_pipe.pipe b/server/tinybird/materializations/events_hourly_no_properties_mv_pipe.pipe new file mode 100644 index 000000000..e3b027513 --- /dev/null +++ b/server/tinybird/materializations/events_hourly_no_properties_mv_pipe.pipe @@ -0,0 +1,21 @@ +DESCRIPTION > + Materializes events into hourly aggregates WITHOUT properties. + Drops properties from GROUP BY so high-cardinality property values + (e.g. unique jobIds) don't prevent row aggregation. + +NODE materialize +SQL > + SELECT + org_id, + env, + customer_id, + event_name, + coalesce(entity_id, '') as entity_id, + toStartOfHour(timestamp) as hour, + sum(toFloat64(coalesce(value, 1))) as total_value, + count() as event_count + FROM events + GROUP BY org_id, env, customer_id, event_name, entity_id, hour + +TYPE materialized +DATASOURCE events_hourly_no_properties_mv diff --git a/server/tinybird/pipes/aggregate_simple.pipe b/server/tinybird/pipes/aggregate_simple.pipe index ad82a7921..a35d63578 100644 --- a/server/tinybird/pipes/aggregate_simple.pipe +++ b/server/tinybird/pipes/aggregate_simple.pipe @@ -18,7 +18,7 @@ SQL > {% end %} event_name, sum(total_value) as total_value - FROM {% if defined(filter_key_0) and String(filter_key_0, '') != '' %}events_hourly_mv{% elif defined(filter_key_1) and String(filter_key_1, '') != '' %}events_hourly_mv{% elif defined(filter_key_2) and String(filter_key_2, '') != '' %}events_hourly_mv{% elif defined(filter_key_3) and String(filter_key_3, '') != '' %}events_hourly_mv{% elif defined(filter_key_4) and String(filter_key_4, '') != '' %}events_hourly_mv{% else %}events_hourly_no_props_mv{% end %} + FROM {% if defined(filter_key_0) and String(filter_key_0, '') != '' %}events_hourly_mv{% elif defined(filter_key_1) and String(filter_key_1, '') != '' %}events_hourly_mv{% elif defined(filter_key_2) and String(filter_key_2, '') != '' %}events_hourly_mv{% elif defined(filter_key_3) and String(filter_key_3, '') != '' %}events_hourly_mv{% elif defined(filter_key_4) and String(filter_key_4, '') != '' %}events_hourly_mv{% else %}events_hourly_no_properties_mv{% end %} WHERE org_id = {{ String(org_id, '') }} AND env = {{ String(env, 'test') }}