comi-logic/cfw-autumn
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 🎸 firecrawl emergency

Browse Source
This commit is contained in:
amianthus
2026-04-05 17:47:33 +01:00
parent b2fd12555c
commit 19e6083b16
5 changed files with 50 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
server/tinybird/connections/firecrawl_backfill_apikeyid.connection Normal file
View File

@@ -0,0 +1,4 @@
TYPE s3
S3_ARN {{ tb_secret("s3_role_arn_firecrawl_backfill_apikeyid_75f684b3") }}
S3_REGION us-west-2
# Learn more at https://www.tinybird.co/docs/forward/get-data-in/connectors/s3#s3-connection-settings

4
server/tinybird/datasources/events.datasource
View File

@@ -21,3 +21,7 @@ SCHEMA >
ENGINE "MergeTree"
ENGINE_PARTITION_KEY "toYYYYMM(timestamp)"
ENGINE_SORTING_KEY "org_id, env, customer_id, event_name, timestamp"
IMPORT_CONNECTION_NAME firecrawl_backfill_apikeyid
IMPORT_BUCKET_URI s3://firecrawl-emergency-backfill-apikeyid/v2/**/*.ndjson
IMPORT_SCHEDULE @auto

20
server/tinybird/materializations/events_hourly_no_properties_mv.datasource Normal file
View File

@@ -0,0 +1,20 @@
DESCRIPTION >
Hourly aggregates WITHOUT properties for fast aggregation queries.
Used by aggregate_simple when no property filters are needed.
Drops the properties column from the GROUP BY so rows with different
properties but the same (org, env, customer, event, entity, hour)
are collapsed into a single row.
SCHEMA >
`org_id` String,
`env` String,
`customer_id` String,
`event_name` String,
`entity_id` String DEFAULT '',
`hour` DateTime,
`total_value` Float64,
`event_count` UInt64
ENGINE "MergeTree"
ENGINE_PARTITION_KEY "toYYYYMM(hour)"
ENGINE_SORTING_KEY "org_id, env, hour, customer_id, entity_id, event_name"

21
server/tinybird/materializations/events_hourly_no_properties_mv_pipe.pipe Normal file
View File

@@ -0,0 +1,21 @@
DESCRIPTION >
Materializes events into hourly aggregates WITHOUT properties.
Drops properties from GROUP BY so high-cardinality property values
(e.g. unique jobIds) don't prevent row aggregation.
NODE materialize
SQL >
SELECT
org_id,
env,
customer_id,
event_name,
coalesce(entity_id, '') as entity_id,
toStartOfHour(timestamp) as hour,
sum(toFloat64(coalesce(value, 1))) as total_value,
count() as event_count
FROM events
GROUP BY org_id, env, customer_id, event_name, entity_id, hour
TYPE materialized
DATASOURCE events_hourly_no_properties_mv

2
server/tinybird/pipes/aggregate_simple.pipe
View File

@@ -18,7 +18,7 @@ SQL >
{% end %}
event_name,
sum(total_value) as total_value
FROM {% if defined(filter_key_0) and String(filter_key_0, '') != '' %}events_hourly_mv{% elif defined(filter_key_1) and String(filter_key_1, '') != '' %}events_hourly_mv{% elif defined(filter_key_2) and String(filter_key_2, '') != '' %}events_hourly_mv{% elif defined(filter_key_3) and String(filter_key_3, '') != '' %}events_hourly_mv{% elif defined(filter_key_4) and String(filter_key_4, '') != '' %}events_hourly_mv{% else %}events_hourly_no_props_mv{% end %}
FROM {% if defined(filter_key_0) and String(filter_key_0, '') != '' %}events_hourly_mv{% elif defined(filter_key_1) and String(filter_key_1, '') != '' %}events_hourly_mv{% elif defined(filter_key_2) and String(filter_key_2, '') != '' %}events_hourly_mv{% elif defined(filter_key_3) and String(filter_key_3, '') != '' %}events_hourly_mv{% elif defined(filter_key_4) and String(filter_key_4, '') != '' %}events_hourly_mv{% else %}events_hourly_no_properties_mv{% end %}
WHERE
org_id = {{ String(org_id, '') }}
AND env = {{ String(env, 'test') }}