feat: 🎸 final scripts and datasources for cdc
This commit is contained in:
amianthus
284
server/tinybird/scripts/backfill_base.ts
Normal file
284
server/tinybird/scripts/backfill_base.ts
Normal file
@@ -0,0 +1,284 @@
|
||||
#!/usr/bin/env bun
|
||||
|
||||
/**
|
||||
* Shared utilities and config for all epoch-ms-chunked Tinybird backfill scripts.
|
||||
*/
|
||||
|
||||
import { execSync } from "child_process";
|
||||
|
||||
// ============================================================================
|
||||
// CONSTANTS
|
||||
// ============================================================================
|
||||
|
||||
export const MAX_RETRIES = 3;
|
||||
export const RETRY_DELAY_MS = 5000;
|
||||
export const DELAY_BETWEEN_CHUNKS_MS = 2000;
|
||||
export const DEFAULT_CHUNK_DAYS = 30;
|
||||
|
||||
// ============================================================================
|
||||
// TYPES
|
||||
// ============================================================================
|
||||
|
||||
export interface Chunk {
|
||||
startEpochMs: number;
|
||||
endEpochMs: number;
|
||||
}
|
||||
|
||||
export interface TableConfig {
|
||||
/** Human-readable label, e.g. "Customers" */
|
||||
label: string;
|
||||
/** Tinybird copy pipe name, e.g. "customers_backfill" */
|
||||
copyPipeName: string;
|
||||
/** Tinybird datasource table name, e.g. "customers" */
|
||||
datasource: string;
|
||||
/** Epoch ms of the earliest known row in Postgres */
|
||||
startEpochMs: number;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// UTILITIES
|
||||
// ============================================================================
|
||||
|
||||
export function execCapture({ cmd }: { cmd: string }): string {
|
||||
try {
|
||||
return execSync(cmd, { encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim();
|
||||
} catch (error: any) {
|
||||
return error.stdout?.trim() ?? "";
|
||||
}
|
||||
}
|
||||
|
||||
const TB_API_URL = process.env.TINYBIRD_API_URL ?? "https://api.us-west-2.aws.tinybird.co";
|
||||
const TB_TOKEN = process.env.TINYBIRD_TOKEN;
|
||||
|
||||
if (!TB_TOKEN) {
|
||||
console.error("TINYBIRD_TOKEN env var is not set. Export it before running backfill scripts.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
/** Query Tinybird SQL API directly — returns parsed rows as an array of objects. */
|
||||
export async function tbSql({ query }: { query: string }): Promise<Record<string, unknown>[]> {
|
||||
const queryWithFormat = `${query.trimEnd()} FORMAT JSONEachRow`;
|
||||
const url = `${TB_API_URL}/v0/sql?q=${encodeURIComponent(queryWithFormat)}`;
|
||||
const res = await fetch(url, {
|
||||
headers: { Authorization: `Bearer ${TB_TOKEN}` },
|
||||
});
|
||||
if (!res.ok) {
|
||||
const body = await res.text();
|
||||
throw new Error(`Tinybird SQL API error ${res.status}: ${body}`);
|
||||
}
|
||||
const text = await res.text();
|
||||
// JSONEachRow returns one JSON object per line
|
||||
return text
|
||||
.split("\n")
|
||||
.map((l) => l.trim())
|
||||
.filter((l) => l.length > 0)
|
||||
.map((l) => JSON.parse(l) as Record<string, unknown>);
|
||||
}
|
||||
|
||||
export function exec({ cmd }: { cmd: string }): void {
|
||||
execSync(cmd, { encoding: "utf-8", stdio: "inherit" });
|
||||
}
|
||||
|
||||
export async function sleep({ ms }: { ms: number }): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/** Extract a single numeric value from the first row/column of a tbSql result. */
|
||||
export function extractNumber({ rows, col }: { rows: Record<string, unknown>[]; col: string }): number | null {
|
||||
const val = rows[0]?.[col];
|
||||
if (val === null || val === undefined) return null;
|
||||
const num = Number(val);
|
||||
return isNaN(num) ? null : num;
|
||||
}
|
||||
|
||||
export function formatEpochMs({ epochMs }: { epochMs: number }): string {
|
||||
return new Date(epochMs).toISOString();
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// TINYBIRD HELPERS
|
||||
// ============================================================================
|
||||
|
||||
/** Returns the MAX created_at among backfill ('read') rows, or null if none. */
|
||||
export async function getMaxCreatedAt({ datasource }: { datasource: string }): Promise<number | null> {
|
||||
const rows = await tbSql({
|
||||
query: `SELECT max(created_at) AS val FROM ${datasource} WHERE __action = 'read'`,
|
||||
});
|
||||
const num = extractNumber({ rows, col: "val" });
|
||||
return num && num > 0 ? num : null;
|
||||
}
|
||||
|
||||
/** Returns count of non-deleted rows in the datasource. */
|
||||
export async function getRowCount({ datasource }: { datasource: string }): Promise<number> {
|
||||
const rows = await tbSql({
|
||||
query: `SELECT count() AS val FROM ${datasource} FINAL WHERE __action != 'delete'`,
|
||||
});
|
||||
return extractNumber({ rows, col: "val" }) ?? 0;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// CHUNK GENERATION (FORWARD — oldest → newest)
|
||||
// ============================================================================
|
||||
|
||||
export function generateChunksForward({
|
||||
startEpochMs,
|
||||
endEpochMs,
|
||||
chunkDays,
|
||||
}: {
|
||||
startEpochMs: number;
|
||||
endEpochMs: number;
|
||||
chunkDays: number;
|
||||
}): Chunk[] {
|
||||
const chunks: Chunk[] = [];
|
||||
const chunkMs = chunkDays * 24 * 60 * 60 * 1000;
|
||||
let current = startEpochMs;
|
||||
|
||||
while (current < endEpochMs) {
|
||||
const next = Math.min(current + chunkMs, endEpochMs);
|
||||
chunks.push({ startEpochMs: current, endEpochMs: next });
|
||||
current = next;
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// COPY JOB EXECUTION
|
||||
// ============================================================================
|
||||
|
||||
export async function waitForCopyJobs(): Promise<void> {
|
||||
const maxAttempts = 60;
|
||||
|
||||
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
||||
const waiting = execCapture({
|
||||
cmd: "tb --cloud job ls --status waiting --kind copy 2>/dev/null | grep -c '^id:' || echo 0",
|
||||
});
|
||||
const working = execCapture({
|
||||
cmd: "tb --cloud job ls --status working --kind copy 2>/dev/null | grep -c '^id:' || echo 0",
|
||||
});
|
||||
const total = parseInt(waiting, 10) + parseInt(working, 10);
|
||||
|
||||
if (total === 0) return;
|
||||
if (attempt === 0) console.log(` Waiting for ${total} existing copy job(s) to complete...`);
|
||||
|
||||
await sleep({ ms: 5000 });
|
||||
}
|
||||
|
||||
throw new Error("Timed out waiting for existing copy jobs to complete");
|
||||
}
|
||||
|
||||
export async function runCopyJob({
|
||||
copyPipeName,
|
||||
startEpochMs,
|
||||
endEpochMs,
|
||||
retries = MAX_RETRIES,
|
||||
}: {
|
||||
copyPipeName: string;
|
||||
startEpochMs: number;
|
||||
endEpochMs: number;
|
||||
retries?: number;
|
||||
}): Promise<boolean> {
|
||||
for (let attempt = 1; attempt <= retries; attempt++) {
|
||||
try {
|
||||
await waitForCopyJobs();
|
||||
exec({
|
||||
cmd: `tb --cloud copy run ${copyPipeName} --param start_epoch_ms="${startEpochMs}" --param end_epoch_ms="${endEpochMs}" --wait`,
|
||||
});
|
||||
return true;
|
||||
} catch (error: any) {
|
||||
const msg = error.message || error.toString();
|
||||
if (attempt < retries) {
|
||||
console.log(` Attempt ${attempt}/${retries} failed. Retrying in ${RETRY_DELAY_MS / 1000}s...`);
|
||||
console.log(` Error: ${msg.substring(0, 200)}`);
|
||||
await sleep({ ms: RETRY_DELAY_MS });
|
||||
} else {
|
||||
console.error(` All ${retries} attempts failed for chunk ${startEpochMs} -> ${endEpochMs}`);
|
||||
console.error(` Error: ${msg}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// CORE BACKFILL RUNNER
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Runs a complete forward (oldest → newest) backfill for a single table config.
|
||||
* Idempotent: resumes from MAX(created_at) of existing 'read' rows.
|
||||
*/
|
||||
export async function runTableBackfill({
|
||||
config,
|
||||
chunkDays,
|
||||
dryRun,
|
||||
}: {
|
||||
config: TableConfig;
|
||||
chunkDays: number;
|
||||
dryRun: boolean;
|
||||
}): Promise<void> {
|
||||
console.log(`\n=== ${config.label} Backfill ===`);
|
||||
|
||||
const tinybirdMax = await getMaxCreatedAt({ datasource: config.datasource });
|
||||
if (tinybirdMax) {
|
||||
console.log(` Resume point: ${tinybirdMax} (${formatEpochMs({ epochMs: tinybirdMax })})`);
|
||||
} else {
|
||||
console.log(" No backfilled rows yet — starting from beginning");
|
||||
}
|
||||
|
||||
const resumePoint = tinybirdMax ? tinybirdMax + 1 : config.startEpochMs;
|
||||
const endPoint = Date.now();
|
||||
|
||||
console.log(` Start: ${formatEpochMs({ epochMs: resumePoint })}`);
|
||||
console.log(` End: ${formatEpochMs({ epochMs: endPoint })}`);
|
||||
|
||||
if (resumePoint >= endPoint) {
|
||||
console.log(` Already complete. Rows in Tinybird: ${await getRowCount({ datasource: config.datasource })}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const chunks = generateChunksForward({ startEpochMs: resumePoint, endEpochMs: endPoint, chunkDays });
|
||||
console.log(` Chunks: ${chunks.length} (${chunkDays} days each)\n`);
|
||||
|
||||
if (dryRun) {
|
||||
chunks.forEach((chunk, i) => {
|
||||
console.log(
|
||||
` ${i + 1}. ${formatEpochMs({ epochMs: chunk.startEpochMs })} -> ${formatEpochMs({ epochMs: chunk.endEpochMs })}`,
|
||||
);
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const chunk = chunks[i];
|
||||
const chunkNum = i + 1;
|
||||
|
||||
console.log(
|
||||
`Chunk ${chunkNum}/${chunks.length}: ${formatEpochMs({ epochMs: chunk.startEpochMs })} -> ${formatEpochMs({ epochMs: chunk.endEpochMs })}`,
|
||||
);
|
||||
|
||||
const startTime = Date.now();
|
||||
const success = await runCopyJob({
|
||||
copyPipeName: config.copyPipeName,
|
||||
startEpochMs: chunk.startEpochMs,
|
||||
endEpochMs: chunk.endEpochMs,
|
||||
});
|
||||
const duration = Math.round((Date.now() - startTime) / 1000);
|
||||
|
||||
if (success) {
|
||||
const count = await getRowCount({ datasource: config.datasource });
|
||||
console.log(` ✓ Chunk ${chunkNum} done in ${duration}s. Rows: ${count}\n`);
|
||||
} else {
|
||||
console.log(` ✗ Chunk ${chunkNum} FAILED after ${duration}s`);
|
||||
console.log(" Re-run this script to resume.\n");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (i < chunks.length - 1) {
|
||||
await sleep({ ms: DELAY_BETWEEN_CHUNKS_MS });
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`=== ${config.label} Backfill Complete. Rows: ${await getRowCount({ datasource: config.datasource })} ===\n`);
|
||||
}
|
||||
303
server/tinybird/scripts/backfill_cli.ts
Normal file
303
server/tinybird/scripts/backfill_cli.ts
Normal file
@@ -0,0 +1,303 @@
|
||||
#!/usr/bin/env bun
|
||||
|
||||
/**
|
||||
* Interactive CLI for backfilling Postgres tables into Tinybird.
|
||||
*
|
||||
* Multi-select which tables to backfill, then runs them sequentially.
|
||||
* All jobs are idempotent — safe to re-run, automatically resumes from progress.
|
||||
*
|
||||
* Usage:
|
||||
* bun scripts/backfill_cli.ts
|
||||
* bun scripts/backfill_cli.ts --chunk-days 14
|
||||
* bun scripts/backfill_cli.ts --dry-run
|
||||
* bun scripts/backfill_cli.ts --all
|
||||
*/
|
||||
|
||||
import * as readline from "readline";
|
||||
import {
|
||||
DEFAULT_CHUNK_DAYS,
|
||||
type TableConfig,
|
||||
runTableBackfill,
|
||||
} from "./backfill_base.js";
|
||||
|
||||
// ============================================================================
|
||||
// TABLE REGISTRY
|
||||
// NOTE: rollovers is NOT listed here — it uses cursor-based pagination.
|
||||
// Run: bun scripts/backfill_rollovers.ts
|
||||
// ============================================================================
|
||||
|
||||
const TABLE_CONFIGS: TableConfig[] = [
|
||||
{
|
||||
label: "customers",
|
||||
copyPipeName: "customers_backfill",
|
||||
datasource: "customers",
|
||||
startEpochMs: 1706987055000,
|
||||
},
|
||||
{
|
||||
label: "invoices",
|
||||
copyPipeName: "invoices_backfill",
|
||||
datasource: "invoices",
|
||||
startEpochMs: 1706987056000,
|
||||
},
|
||||
{
|
||||
label: "organizations",
|
||||
copyPipeName: "organizations_backfill",
|
||||
datasource: "organizations",
|
||||
startEpochMs: 1737543151220,
|
||||
},
|
||||
{
|
||||
label: "customer_products",
|
||||
copyPipeName: "customer_products_backfill",
|
||||
datasource: "customer_products",
|
||||
startEpochMs: 1677713742000,
|
||||
},
|
||||
{
|
||||
label: "customer_entitlements",
|
||||
copyPipeName: "customer_entitlements_backfill",
|
||||
datasource: "customer_entitlements",
|
||||
startEpochMs: 1738268254191,
|
||||
},
|
||||
{
|
||||
label: "customer_prices",
|
||||
copyPipeName: "customer_prices_backfill",
|
||||
datasource: "customer_prices",
|
||||
startEpochMs: 1738341655109,
|
||||
},
|
||||
{
|
||||
label: "replaceables",
|
||||
copyPipeName: "replaceables_backfill",
|
||||
datasource: "replaceables",
|
||||
startEpochMs: 1751360953240,
|
||||
},
|
||||
{
|
||||
label: "entitlements",
|
||||
copyPipeName: "entitlements_backfill",
|
||||
datasource: "entitlements",
|
||||
startEpochMs: 1737570713388,
|
||||
},
|
||||
{
|
||||
label: "free_trials",
|
||||
copyPipeName: "free_trials_backfill",
|
||||
datasource: "free_trials",
|
||||
startEpochMs: 1738168893927,
|
||||
},
|
||||
{
|
||||
label: "entities",
|
||||
copyPipeName: "entities_backfill",
|
||||
datasource: "entities",
|
||||
startEpochMs: 1743685142432,
|
||||
},
|
||||
{
|
||||
label: "subscriptions",
|
||||
copyPipeName: "subscriptions_backfill",
|
||||
datasource: "subscriptions",
|
||||
startEpochMs: 1744118448491,
|
||||
},
|
||||
{
|
||||
label: "features",
|
||||
copyPipeName: "features_backfill",
|
||||
datasource: "features",
|
||||
startEpochMs: 1737570301197,
|
||||
},
|
||||
{
|
||||
label: "prices",
|
||||
copyPipeName: "prices_backfill",
|
||||
datasource: "prices",
|
||||
startEpochMs: 1737570713388,
|
||||
},
|
||||
{
|
||||
label: "products",
|
||||
copyPipeName: "products_backfill",
|
||||
datasource: "products",
|
||||
startEpochMs: 1737570326143,
|
||||
},
|
||||
// rollovers intentionally excluded — use backfill_rollovers.ts instead
|
||||
];
|
||||
|
||||
// ============================================================================
|
||||
// ARGUMENT PARSING
|
||||
// ============================================================================
|
||||
|
||||
interface Args {
|
||||
chunkDays: number;
|
||||
dryRun: boolean;
|
||||
all: boolean;
|
||||
}
|
||||
|
||||
function parseArgs(): Args {
|
||||
const args: Args = {
|
||||
chunkDays: DEFAULT_CHUNK_DAYS,
|
||||
dryRun: false,
|
||||
all: false,
|
||||
};
|
||||
|
||||
for (let i = 2; i < process.argv.length; i++) {
|
||||
const arg = process.argv[i];
|
||||
if (arg === "--chunk-days" && process.argv[i + 1]) {
|
||||
args.chunkDays = parseInt(process.argv[++i], 10);
|
||||
} else if (arg === "--dry-run") {
|
||||
args.dryRun = true;
|
||||
} else if (arg === "--all") {
|
||||
args.all = true;
|
||||
} else if (arg === "--help" || arg === "-h") {
|
||||
console.log(`
|
||||
Interactive Tinybird backfill CLI
|
||||
|
||||
Usage: bun scripts/backfill_cli.ts [options]
|
||||
|
||||
Options:
|
||||
--chunk-days <n> Days per chunk (default: ${DEFAULT_CHUNK_DAYS})
|
||||
--dry-run Show chunks without executing copy jobs
|
||||
--all Skip prompt, backfill all tables
|
||||
--help, -h Show this help message
|
||||
|
||||
Available tables:
|
||||
${TABLE_CONFIGS.map((t, i) => ` ${i + 1}. ${t.label}`).join("\n")}
|
||||
|
||||
Note: rollovers uses cursor-based pagination — run separately:
|
||||
bun scripts/backfill_rollovers.ts
|
||||
`);
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
return args;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// MULTI-SELECT PROMPT
|
||||
// ============================================================================
|
||||
|
||||
async function multiSelect({
|
||||
items,
|
||||
prompt,
|
||||
}: {
|
||||
items: string[];
|
||||
prompt: string;
|
||||
}): Promise<number[]> {
|
||||
const selected = new Set<number>();
|
||||
|
||||
// Pre-select all
|
||||
items.forEach((_, i) => selected.add(i));
|
||||
|
||||
const rl = readline.createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
});
|
||||
|
||||
// Keys: 1-9 for indices 0-8, then a-z for indices 9-34
|
||||
// (a/A are reserved for select-all, n/N for select-none — skip those)
|
||||
const KEYS = "123456789bcdefghijklmopqrstuvwxyz";
|
||||
|
||||
const keyForIndex = (i: number) => KEYS[i] ?? "?";
|
||||
|
||||
const renderMenu = () => {
|
||||
console.clear();
|
||||
console.log(prompt);
|
||||
console.log("(key to toggle, A to select all, N to select none, Enter to confirm)\n");
|
||||
items.forEach((item, i) => {
|
||||
const checked = selected.has(i) ? "[x]" : "[ ]";
|
||||
console.log(` ${checked} ${keyForIndex(i)}. ${item}`);
|
||||
});
|
||||
console.log("");
|
||||
};
|
||||
|
||||
return new Promise((resolve) => {
|
||||
renderMenu();
|
||||
|
||||
// Use raw mode for keypress detection
|
||||
if (process.stdin.isTTY) {
|
||||
process.stdin.setRawMode(true);
|
||||
}
|
||||
|
||||
const onKeypress = (chunk: Buffer) => {
|
||||
const key = chunk.toString().toLowerCase();
|
||||
|
||||
if (key === "\r" || key === "\n") {
|
||||
// Enter — confirm
|
||||
process.stdin.removeListener("data", onKeypress);
|
||||
if (process.stdin.isTTY) {
|
||||
process.stdin.setRawMode(false);
|
||||
}
|
||||
rl.close();
|
||||
console.clear();
|
||||
resolve([...selected].sort((a, b) => a - b));
|
||||
return;
|
||||
}
|
||||
|
||||
if (key === "a") {
|
||||
items.forEach((_, i) => selected.add(i));
|
||||
renderMenu();
|
||||
return;
|
||||
}
|
||||
|
||||
if (key === "n") {
|
||||
selected.clear();
|
||||
renderMenu();
|
||||
return;
|
||||
}
|
||||
|
||||
if (key === "\x03") {
|
||||
// Ctrl+C
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Map key to index
|
||||
const idx = KEYS.indexOf(key);
|
||||
if (idx !== -1 && idx < items.length) {
|
||||
if (selected.has(idx)) {
|
||||
selected.delete(idx);
|
||||
} else {
|
||||
selected.add(idx);
|
||||
}
|
||||
renderMenu();
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
process.stdin.on("data", onKeypress);
|
||||
});
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// MAIN
|
||||
// ============================================================================
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs();
|
||||
|
||||
let selectedConfigs: TableConfig[];
|
||||
|
||||
if (args.all) {
|
||||
selectedConfigs = TABLE_CONFIGS;
|
||||
console.log("--all flag set: backfilling all tables.\n");
|
||||
} else {
|
||||
const selectedIndices = await multiSelect({
|
||||
items: TABLE_CONFIGS.map((t) => t.label),
|
||||
prompt: "Select tables to backfill:",
|
||||
});
|
||||
|
||||
if (selectedIndices.length === 0) {
|
||||
console.log("No tables selected. Exiting.");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
selectedConfigs = selectedIndices.map((i) => TABLE_CONFIGS[i]);
|
||||
}
|
||||
|
||||
console.log(`\nBackfilling ${selectedConfigs.length} table(s): ${selectedConfigs.map((t) => t.label).join(", ")}`);
|
||||
console.log(`Chunk size: ${args.chunkDays} days`);
|
||||
if (args.dryRun) console.log("DRY RUN — no copy jobs will be executed.\n");
|
||||
|
||||
for (const config of selectedConfigs) {
|
||||
await runTableBackfill({ config, chunkDays: args.chunkDays, dryRun: args.dryRun });
|
||||
}
|
||||
|
||||
console.log("\n==========================================");
|
||||
console.log("All selected backfills complete.");
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error("Fatal error:", error);
|
||||
process.exit(1);
|
||||
});
|
||||
322
server/tinybird/scripts/backfill_customers.ts
Normal file
322
server/tinybird/scripts/backfill_customers.ts
Normal file
@@ -0,0 +1,322 @@
|
||||
#!/usr/bin/env bun
|
||||
|
||||
/**
|
||||
* Idempotent backfill script for customers from Postgres to Tinybird.
|
||||
*
|
||||
* Chunks by created_at epoch ms. Fills oldest → newest.
|
||||
* Safe to re-run — uses MAX(created_at) in Tinybird to find resume point.
|
||||
*
|
||||
* Usage:
|
||||
* bun scripts/backfill_customers.ts
|
||||
* bun scripts/backfill_customers.ts --chunk-days 30
|
||||
* bun scripts/backfill_customers.ts --dry-run
|
||||
*/
|
||||
|
||||
import { execSync } from "child_process";
|
||||
|
||||
// ============================================================================
|
||||
// CONFIGURATION
|
||||
// ============================================================================
|
||||
|
||||
// Earliest possible customer (epoch ms). Adjust if needed.
|
||||
const START_EPOCH_MS = 1706987055000;
|
||||
|
||||
// Latest epoch ms to backfill up to — set to "now" at runtime
|
||||
const END_EPOCH_MS = Date.now();
|
||||
|
||||
const DEFAULT_CHUNK_DAYS = 30;
|
||||
const MAX_RETRIES = 3;
|
||||
const RETRY_DELAY_MS = 5000;
|
||||
const DELAY_BETWEEN_CHUNKS_MS = 2000;
|
||||
|
||||
// ============================================================================
|
||||
// TYPES
|
||||
// ============================================================================
|
||||
|
||||
interface Args {
|
||||
chunkDays: number;
|
||||
dryRun: boolean;
|
||||
}
|
||||
|
||||
interface Chunk {
|
||||
startEpochMs: number;
|
||||
endEpochMs: number;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// ARGUMENT PARSING
|
||||
// ============================================================================
|
||||
|
||||
function parseArgs(): Args {
|
||||
const args: Args = {
|
||||
chunkDays: DEFAULT_CHUNK_DAYS,
|
||||
dryRun: false,
|
||||
};
|
||||
|
||||
for (let i = 2; i < process.argv.length; i++) {
|
||||
const arg = process.argv[i];
|
||||
if (arg === "--chunk-days" && process.argv[i + 1]) {
|
||||
args.chunkDays = parseInt(process.argv[++i], 10);
|
||||
} else if (arg === "--dry-run") {
|
||||
args.dryRun = true;
|
||||
} else if (arg === "--help" || arg === "-h") {
|
||||
console.log(`
|
||||
Idempotent backfill script for customers from Postgres to Tinybird
|
||||
|
||||
Usage: bun scripts/backfill_customers.ts [options]
|
||||
|
||||
Options:
|
||||
--chunk-days <n> Days per chunk (default: ${DEFAULT_CHUNK_DAYS})
|
||||
--dry-run Show what would be done without executing
|
||||
--help, -h Show this help message
|
||||
`);
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
return args;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// UTILITIES
|
||||
// ============================================================================
|
||||
|
||||
function execCapture({ cmd }: { cmd: string }): string {
|
||||
try {
|
||||
return execSync(cmd, { encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim();
|
||||
} catch (error: any) {
|
||||
return error.stdout?.trim() ?? "";
|
||||
}
|
||||
}
|
||||
|
||||
function tbSql({ query }: { query: string }): string {
|
||||
const escaped = query.replace(/"/g, '\\"');
|
||||
try {
|
||||
const result = execSync(`tb --cloud sql "${escaped}"`, {
|
||||
encoding: "utf-8",
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
return result.trim();
|
||||
} catch (error: any) {
|
||||
return error.stdout?.trim() ?? "";
|
||||
}
|
||||
}
|
||||
|
||||
function exec({ cmd }: { cmd: string }): void {
|
||||
execSync(cmd, { encoding: "utf-8", stdio: "inherit" });
|
||||
}
|
||||
|
||||
async function sleep({ ms }: { ms: number }): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function extractNumber({ result }: { result: string }): number | null {
|
||||
for (const line of result.split("\n")) {
|
||||
const cleaned = line.trim();
|
||||
const num = Number(cleaned);
|
||||
if (!isNaN(num) && cleaned !== "" && String(Math.round(num)) === cleaned) {
|
||||
return num;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function formatEpochMs({ epochMs }: { epochMs: number }): string {
|
||||
return new Date(epochMs).toISOString();
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// TINYBIRD QUERIES
|
||||
// ============================================================================
|
||||
|
||||
function getMaxCreatedAtInTinybird(): number | null {
|
||||
console.log("Querying Tinybird for backfill progress...");
|
||||
const result = tbSql({
|
||||
query: "SELECT max(created_at) FROM customers WHERE __action = 'read'",
|
||||
});
|
||||
const num = extractNumber({ result });
|
||||
if (num && num > 0) {
|
||||
console.log(` Tinybird MAX created_at (backfill rows): ${num} (${formatEpochMs({ epochMs: num })})`);
|
||||
return num;
|
||||
}
|
||||
console.log(" Tinybird MAX created_at: No backfilled rows yet");
|
||||
return null;
|
||||
}
|
||||
|
||||
function getCustomerCount(): number {
|
||||
const result = tbSql({ query: "SELECT count() FROM customers FINAL WHERE __action != 'delete'" });
|
||||
return extractNumber({ result }) ?? 0;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// CHUNK GENERATION (FORWARD)
|
||||
// ============================================================================
|
||||
|
||||
function generateChunksForward({
|
||||
startEpochMs,
|
||||
endEpochMs,
|
||||
chunkDays,
|
||||
}: {
|
||||
startEpochMs: number;
|
||||
endEpochMs: number;
|
||||
chunkDays: number;
|
||||
}): Chunk[] {
|
||||
const chunks: Chunk[] = [];
|
||||
const chunkMs = chunkDays * 24 * 60 * 60 * 1000;
|
||||
let current = startEpochMs;
|
||||
|
||||
while (current < endEpochMs) {
|
||||
const next = Math.min(current + chunkMs, endEpochMs);
|
||||
chunks.push({ startEpochMs: current, endEpochMs: next });
|
||||
current = next;
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// COPY JOB EXECUTION
|
||||
// ============================================================================
|
||||
|
||||
async function waitForCopyJobs(): Promise<void> {
|
||||
const maxAttempts = 60;
|
||||
|
||||
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
||||
const waiting = execCapture({
|
||||
cmd: "tb --cloud job ls --status waiting --kind copy 2>/dev/null | grep -c '^id:' || echo 0",
|
||||
});
|
||||
const working = execCapture({
|
||||
cmd: "tb --cloud job ls --status working --kind copy 2>/dev/null | grep -c '^id:' || echo 0",
|
||||
});
|
||||
const total = parseInt(waiting, 10) + parseInt(working, 10);
|
||||
|
||||
if (total === 0) return;
|
||||
if (attempt === 0) console.log(` Waiting for ${total} existing copy job(s) to complete...`);
|
||||
|
||||
await sleep({ ms: 5000 });
|
||||
}
|
||||
|
||||
throw new Error("Timed out waiting for existing copy jobs to complete");
|
||||
}
|
||||
|
||||
async function runCopyJob({
|
||||
startEpochMs,
|
||||
endEpochMs,
|
||||
retries = MAX_RETRIES,
|
||||
}: {
|
||||
startEpochMs: number;
|
||||
endEpochMs: number;
|
||||
retries?: number;
|
||||
}): Promise<boolean> {
|
||||
for (let attempt = 1; attempt <= retries; attempt++) {
|
||||
try {
|
||||
await waitForCopyJobs();
|
||||
exec({
|
||||
cmd: `tb --cloud copy run customers_backfill --param start_epoch_ms="${startEpochMs}" --param end_epoch_ms="${endEpochMs}" --wait`,
|
||||
});
|
||||
return true;
|
||||
} catch (error: any) {
|
||||
const msg = error.message || error.toString();
|
||||
if (attempt < retries) {
|
||||
console.log(` Attempt ${attempt}/${retries} failed. Retrying in ${RETRY_DELAY_MS / 1000}s...`);
|
||||
console.log(` Error: ${msg.substring(0, 200)}`);
|
||||
await sleep({ ms: RETRY_DELAY_MS });
|
||||
} else {
|
||||
console.error(` All ${retries} attempts failed for chunk ${startEpochMs} -> ${endEpochMs}`);
|
||||
console.error(` Error: ${msg}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// MAIN
|
||||
// ============================================================================
|
||||
|
||||
async function main() {
|
||||
console.log("=== Customers Backfill Script ===\n");
|
||||
|
||||
const args = parseArgs();
|
||||
|
||||
// 1. Find resume point from Tinybird
|
||||
const tinybirdMax = getMaxCreatedAtInTinybird();
|
||||
|
||||
// Resume from just after the last backfilled row, or from the very beginning
|
||||
const resumePoint = tinybirdMax ? tinybirdMax + 1 : START_EPOCH_MS;
|
||||
const endPoint = END_EPOCH_MS;
|
||||
|
||||
console.log(`\nResume point: ${resumePoint} (${formatEpochMs({ epochMs: resumePoint })})`);
|
||||
console.log(`End point: ${endPoint} (${formatEpochMs({ epochMs: endPoint })})`);
|
||||
|
||||
// 2. Check if done
|
||||
if (resumePoint >= endPoint) {
|
||||
console.log("\n✓ Backfill complete! Nothing more to do.");
|
||||
console.log(` Total customers in Tinybird: ${getCustomerCount()}`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// 3. Generate chunks
|
||||
const chunks = generateChunksForward({
|
||||
startEpochMs: resumePoint,
|
||||
endEpochMs: endPoint,
|
||||
chunkDays: args.chunkDays,
|
||||
});
|
||||
|
||||
console.log(`\nGenerated ${chunks.length} chunks (${args.chunkDays} days each)`);
|
||||
console.log(`Direction: oldest → newest\n`);
|
||||
|
||||
// 4. Dry run
|
||||
if (args.dryRun) {
|
||||
console.log("DRY RUN - Would process these chunks:\n");
|
||||
chunks.forEach((chunk, i) => {
|
||||
console.log(
|
||||
` ${i + 1}. ${formatEpochMs({ epochMs: chunk.startEpochMs })} -> ${formatEpochMs({ epochMs: chunk.endEpochMs })}`,
|
||||
);
|
||||
});
|
||||
console.log(`\nTotal: ${chunks.length} chunks`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// 5. Process chunks
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const chunk = chunks[i];
|
||||
const chunkNum = i + 1;
|
||||
|
||||
console.log(
|
||||
`=== Chunk ${chunkNum}/${chunks.length}: ${formatEpochMs({ epochMs: chunk.startEpochMs })} -> ${formatEpochMs({ epochMs: chunk.endEpochMs })} ===`,
|
||||
);
|
||||
|
||||
const startTime = Date.now();
|
||||
const success = await runCopyJob({
|
||||
startEpochMs: chunk.startEpochMs,
|
||||
endEpochMs: chunk.endEpochMs,
|
||||
});
|
||||
const duration = Math.round((Date.now() - startTime) / 1000);
|
||||
|
||||
if (success) {
|
||||
const count = getCustomerCount();
|
||||
console.log(`✓ Chunk ${chunkNum} COMPLETE in ${duration}s. Customers in Tinybird: ${count}\n`);
|
||||
} else {
|
||||
console.log(`✗ Chunk ${chunkNum} FAILED after ${duration}s\n`);
|
||||
console.log("To resume, simply re-run:");
|
||||
console.log(" bun scripts/backfill_customers.ts\n");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (i < chunks.length - 1) {
|
||||
await sleep({ ms: DELAY_BETWEEN_CHUNKS_MS });
|
||||
}
|
||||
}
|
||||
|
||||
// 6. Summary
|
||||
console.log("==========================================");
|
||||
console.log("=== Backfill Complete ===");
|
||||
console.log(`Total customers in Tinybird: ${getCustomerCount()}`);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error("Fatal error:", error);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -124,7 +124,7 @@ function exec(cmd: string, silent = false): string {
|
||||
|
||||
function execCapture(cmd: string): string {
|
||||
try {
|
||||
return execSync(cmd, { encoding: "utf-8", stderr: "pipe" }).trim();
|
||||
return execSync(cmd, { encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim();
|
||||
} catch (error: any) {
|
||||
return error.stdout?.trim() ?? "";
|
||||
}
|
||||
|
||||
331
server/tinybird/scripts/backfill_invoices.ts
Normal file
331
server/tinybird/scripts/backfill_invoices.ts
Normal file
@@ -0,0 +1,331 @@
|
||||
#!/usr/bin/env bun
|
||||
|
||||
/**
|
||||
* Idempotent backfill script for invoices from Postgres to Tinybird.
|
||||
*
|
||||
* Chunks by created_at epoch ms. Fills oldest → newest.
|
||||
* Safe to re-run — uses MAX(created_at) in Tinybird to find resume point.
|
||||
*
|
||||
* Usage:
|
||||
* bun scripts/backfill_invoices.ts
|
||||
* bun scripts/backfill_invoices.ts --chunk-days 30
|
||||
* bun scripts/backfill_invoices.ts --dry-run
|
||||
*/
|
||||
|
||||
import { execSync } from "child_process";
|
||||
|
||||
// ============================================================================
|
||||
// CONFIGURATION
|
||||
// ============================================================================
|
||||
|
||||
// Earliest possible invoice (epoch ms). Adjust if needed.
|
||||
const START_EPOCH_MS = 1706987056000;
|
||||
|
||||
// Latest epoch ms to backfill up to — set to "now" at runtime
|
||||
const END_EPOCH_MS = Date.now();
|
||||
|
||||
const DEFAULT_CHUNK_DAYS = 30;
|
||||
const MAX_RETRIES = 3;
|
||||
const RETRY_DELAY_MS = 5000;
|
||||
const DELAY_BETWEEN_CHUNKS_MS = 2000;
|
||||
|
||||
// ============================================================================
|
||||
// TYPES
|
||||
// ============================================================================
|
||||
|
||||
interface Args {
|
||||
chunkDays: number;
|
||||
dryRun: boolean;
|
||||
}
|
||||
|
||||
interface Chunk {
|
||||
startEpochMs: number;
|
||||
endEpochMs: number;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// ARGUMENT PARSING
|
||||
// ============================================================================
|
||||
|
||||
function parseArgs(): Args {
|
||||
const args: Args = {
|
||||
chunkDays: DEFAULT_CHUNK_DAYS,
|
||||
dryRun: false,
|
||||
};
|
||||
|
||||
for (let i = 2; i < process.argv.length; i++) {
|
||||
const arg = process.argv[i];
|
||||
if (arg === "--chunk-days" && process.argv[i + 1]) {
|
||||
args.chunkDays = parseInt(process.argv[++i], 10);
|
||||
} else if (arg === "--dry-run") {
|
||||
args.dryRun = true;
|
||||
} else if (arg === "--help" || arg === "-h") {
|
||||
console.log(`
|
||||
Idempotent backfill script for invoices from Postgres to Tinybird
|
||||
|
||||
Usage: bun scripts/backfill_invoices.ts [options]
|
||||
|
||||
Options:
|
||||
--chunk-days <n> Days per chunk (default: ${DEFAULT_CHUNK_DAYS})
|
||||
--dry-run Show what would be done without executing
|
||||
--help, -h Show this help message
|
||||
|
||||
How it works:
|
||||
1. Queries Tinybird for MAX(created_at) to find resume point
|
||||
2. Generates chunks from resume point UP to now
|
||||
3. Runs each chunk as a Tinybird copy job
|
||||
|
||||
Idempotency:
|
||||
- Safe to re-run at any time
|
||||
- Automatically resumes from where it left off
|
||||
`);
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
return args;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// UTILITIES
|
||||
// ============================================================================
|
||||
|
||||
function execCapture({ cmd }: { cmd: string }): string {
|
||||
try {
|
||||
return execSync(cmd, { encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim();
|
||||
} catch (error: any) {
|
||||
return error.stdout?.trim() ?? "";
|
||||
}
|
||||
}
|
||||
|
||||
function tbSql({ query }: { query: string }): string {
|
||||
const escaped = query.replace(/"/g, '\\"');
|
||||
try {
|
||||
const result = execSync(`tb --cloud sql "${escaped}"`, {
|
||||
encoding: "utf-8",
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
return result.trim();
|
||||
} catch (error: any) {
|
||||
return error.stdout?.trim() ?? "";
|
||||
}
|
||||
}
|
||||
|
||||
function exec({ cmd }: { cmd: string }): void {
|
||||
execSync(cmd, { encoding: "utf-8", stdio: "inherit" });
|
||||
}
|
||||
|
||||
async function sleep({ ms }: { ms: number }): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function extractNumber({ result }: { result: string }): number | null {
|
||||
for (const line of result.split("\n")) {
|
||||
const cleaned = line.trim();
|
||||
const num = Number(cleaned);
|
||||
if (!isNaN(num) && cleaned !== "" && String(Math.round(num)) === cleaned) {
|
||||
return num;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function formatEpochMs({ epochMs }: { epochMs: number }): string {
|
||||
return new Date(epochMs).toISOString();
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// TINYBIRD QUERIES
|
||||
// ============================================================================
|
||||
|
||||
function getMaxCreatedAtInTinybird(): number | null {
|
||||
console.log("Querying Tinybird for backfill progress...");
|
||||
const result = tbSql({
|
||||
query: "SELECT max(created_at) FROM invoices WHERE __action = 'read'",
|
||||
});
|
||||
const num = extractNumber({ result });
|
||||
if (num && num > 0) {
|
||||
console.log(` Tinybird MAX created_at (backfill rows): ${num} (${formatEpochMs({ epochMs: num })})`);
|
||||
return num;
|
||||
}
|
||||
console.log(" Tinybird MAX created_at: No backfilled rows yet");
|
||||
return null;
|
||||
}
|
||||
|
||||
function getInvoiceCount(): number {
|
||||
const result = tbSql({ query: "SELECT count() FROM invoices FINAL WHERE __action != 'delete'" });
|
||||
return extractNumber({ result }) ?? 0;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// CHUNK GENERATION (FORWARD)
|
||||
// ============================================================================
|
||||
|
||||
function generateChunksForward({
|
||||
startEpochMs,
|
||||
endEpochMs,
|
||||
chunkDays,
|
||||
}: {
|
||||
startEpochMs: number;
|
||||
endEpochMs: number;
|
||||
chunkDays: number;
|
||||
}): Chunk[] {
|
||||
const chunks: Chunk[] = [];
|
||||
const chunkMs = chunkDays * 24 * 60 * 60 * 1000;
|
||||
let current = startEpochMs;
|
||||
|
||||
while (current < endEpochMs) {
|
||||
const next = Math.min(current + chunkMs, endEpochMs);
|
||||
chunks.push({ startEpochMs: current, endEpochMs: next });
|
||||
current = next;
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// COPY JOB EXECUTION
|
||||
// ============================================================================
|
||||
|
||||
async function waitForCopyJobs(): Promise<void> {
|
||||
const maxAttempts = 60;
|
||||
|
||||
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
||||
const waiting = execCapture({
|
||||
cmd: "tb --cloud job ls --status waiting --kind copy 2>/dev/null | grep -c '^id:' || echo 0",
|
||||
});
|
||||
const working = execCapture({
|
||||
cmd: "tb --cloud job ls --status working --kind copy 2>/dev/null | grep -c '^id:' || echo 0",
|
||||
});
|
||||
const total = parseInt(waiting, 10) + parseInt(working, 10);
|
||||
|
||||
if (total === 0) return;
|
||||
if (attempt === 0) console.log(` Waiting for ${total} existing copy job(s) to complete...`);
|
||||
|
||||
await sleep({ ms: 5000 });
|
||||
}
|
||||
|
||||
throw new Error("Timed out waiting for existing copy jobs to complete");
|
||||
}
|
||||
|
||||
async function runCopyJob({
|
||||
startEpochMs,
|
||||
endEpochMs,
|
||||
retries = MAX_RETRIES,
|
||||
}: {
|
||||
startEpochMs: number;
|
||||
endEpochMs: number;
|
||||
retries?: number;
|
||||
}): Promise<boolean> {
|
||||
for (let attempt = 1; attempt <= retries; attempt++) {
|
||||
try {
|
||||
await waitForCopyJobs();
|
||||
exec({
|
||||
cmd: `tb --cloud copy run invoices_backfill --param start_epoch_ms="${startEpochMs}" --param end_epoch_ms="${endEpochMs}" --wait`,
|
||||
});
|
||||
return true;
|
||||
} catch (error: any) {
|
||||
const msg = error.message || error.toString();
|
||||
if (attempt < retries) {
|
||||
console.log(` Attempt ${attempt}/${retries} failed. Retrying in ${RETRY_DELAY_MS / 1000}s...`);
|
||||
console.log(` Error: ${msg.substring(0, 200)}`);
|
||||
await sleep({ ms: RETRY_DELAY_MS });
|
||||
} else {
|
||||
console.error(` All ${retries} attempts failed for chunk ${startEpochMs} -> ${endEpochMs}`);
|
||||
console.error(` Error: ${msg}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// MAIN
|
||||
// ============================================================================
|
||||
|
||||
async function main() {
|
||||
console.log("=== Invoices Backfill Script ===\n");
|
||||
|
||||
const args = parseArgs();
|
||||
|
||||
// 1. Find resume point from Tinybird
|
||||
const tinybirdMax = getMaxCreatedAtInTinybird();
|
||||
|
||||
// Resume from just after the last backfilled row, or from the very beginning
|
||||
const resumePoint = tinybirdMax ? tinybirdMax + 1 : START_EPOCH_MS;
|
||||
const endPoint = END_EPOCH_MS;
|
||||
|
||||
console.log(`\nResume point: ${resumePoint} (${formatEpochMs({ epochMs: resumePoint })})`);
|
||||
console.log(`End point: ${endPoint} (${formatEpochMs({ epochMs: endPoint })})`);
|
||||
|
||||
// 2. Check if done
|
||||
if (resumePoint >= endPoint) {
|
||||
console.log("\n✓ Backfill complete! Nothing more to do.");
|
||||
console.log(` Total invoices in Tinybird: ${getInvoiceCount()}`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// 3. Generate chunks
|
||||
const chunks = generateChunksForward({
|
||||
startEpochMs: resumePoint,
|
||||
endEpochMs: endPoint,
|
||||
chunkDays: args.chunkDays,
|
||||
});
|
||||
|
||||
console.log(`\nGenerated ${chunks.length} chunks (${args.chunkDays} days each)`);
|
||||
console.log(`Direction: oldest → newest\n`);
|
||||
|
||||
// 4. Dry run
|
||||
if (args.dryRun) {
|
||||
console.log("DRY RUN - Would process these chunks:\n");
|
||||
chunks.forEach((chunk, i) => {
|
||||
console.log(
|
||||
` ${i + 1}. ${formatEpochMs({ epochMs: chunk.startEpochMs })} -> ${formatEpochMs({ epochMs: chunk.endEpochMs })}`,
|
||||
);
|
||||
});
|
||||
console.log(`\nTotal: ${chunks.length} chunks`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// 5. Process chunks
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const chunk = chunks[i];
|
||||
const chunkNum = i + 1;
|
||||
|
||||
console.log(
|
||||
`=== Chunk ${chunkNum}/${chunks.length}: ${formatEpochMs({ epochMs: chunk.startEpochMs })} -> ${formatEpochMs({ epochMs: chunk.endEpochMs })} ===`,
|
||||
);
|
||||
|
||||
const startTime = Date.now();
|
||||
const success = await runCopyJob({
|
||||
startEpochMs: chunk.startEpochMs,
|
||||
endEpochMs: chunk.endEpochMs,
|
||||
});
|
||||
const duration = Math.round((Date.now() - startTime) / 1000);
|
||||
|
||||
if (success) {
|
||||
const count = getInvoiceCount();
|
||||
console.log(`✓ Chunk ${chunkNum} COMPLETE in ${duration}s. Invoices in Tinybird: ${count}\n`);
|
||||
} else {
|
||||
console.log(`✗ Chunk ${chunkNum} FAILED after ${duration}s\n`);
|
||||
console.log("To resume, simply re-run:");
|
||||
console.log(" bun scripts/backfill_invoices.ts\n");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (i < chunks.length - 1) {
|
||||
await sleep({ ms: DELAY_BETWEEN_CHUNKS_MS });
|
||||
}
|
||||
}
|
||||
|
||||
// 6. Summary
|
||||
console.log("==========================================");
|
||||
console.log("=== Backfill Complete ===");
|
||||
console.log(`Total invoices in Tinybird: ${getInvoiceCount()}`);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error("Fatal error:", error);
|
||||
process.exit(1);
|
||||
});
|
||||
325
server/tinybird/scripts/backfill_organizations.ts
Normal file
325
server/tinybird/scripts/backfill_organizations.ts
Normal file
@@ -0,0 +1,325 @@
|
||||
#!/usr/bin/env bun
|
||||
|
||||
/**
|
||||
* Idempotent backfill script for organizations from Postgres to Tinybird.
|
||||
*
|
||||
* Chunks by created_at epoch ms. Fills oldest → newest.
|
||||
* Safe to re-run — uses MAX(created_at) in Tinybird to find resume point.
|
||||
*
|
||||
* NOTE: organizations.created_at is a numeric column (epoch ms), same as invoices/customers.
|
||||
* Organizations without created_at will be missed — use a full (no-chunk) sweep if needed.
|
||||
*
|
||||
* Usage:
|
||||
* bun scripts/backfill_organizations.ts
|
||||
* bun scripts/backfill_organizations.ts --chunk-days 30
|
||||
* bun scripts/backfill_organizations.ts --dry-run
|
||||
*/
|
||||
|
||||
import { execSync } from "child_process";
|
||||
|
||||
// ============================================================================
|
||||
// CONFIGURATION
|
||||
// ============================================================================
|
||||
|
||||
// Earliest possible organization (epoch ms). Adjust if needed.
|
||||
const START_EPOCH_MS = 1737543151220;
|
||||
|
||||
// Latest epoch ms to backfill up to — set to "now" at runtime
|
||||
const END_EPOCH_MS = Date.now();
|
||||
|
||||
const DEFAULT_CHUNK_DAYS = 30;
|
||||
const MAX_RETRIES = 3;
|
||||
const RETRY_DELAY_MS = 5000;
|
||||
const DELAY_BETWEEN_CHUNKS_MS = 2000;
|
||||
|
||||
// ============================================================================
|
||||
// TYPES
|
||||
// ============================================================================
|
||||
|
||||
interface Args {
|
||||
chunkDays: number;
|
||||
dryRun: boolean;
|
||||
}
|
||||
|
||||
interface Chunk {
|
||||
startEpochMs: number;
|
||||
endEpochMs: number;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// ARGUMENT PARSING
|
||||
// ============================================================================
|
||||
|
||||
function parseArgs(): Args {
|
||||
const args: Args = {
|
||||
chunkDays: DEFAULT_CHUNK_DAYS,
|
||||
dryRun: false,
|
||||
};
|
||||
|
||||
for (let i = 2; i < process.argv.length; i++) {
|
||||
const arg = process.argv[i];
|
||||
if (arg === "--chunk-days" && process.argv[i + 1]) {
|
||||
args.chunkDays = parseInt(process.argv[++i], 10);
|
||||
} else if (arg === "--dry-run") {
|
||||
args.dryRun = true;
|
||||
} else if (arg === "--help" || arg === "-h") {
|
||||
console.log(`
|
||||
Idempotent backfill script for organizations from Postgres to Tinybird
|
||||
|
||||
Usage: bun scripts/backfill_organizations.ts [options]
|
||||
|
||||
Options:
|
||||
--chunk-days <n> Days per chunk (default: ${DEFAULT_CHUNK_DAYS})
|
||||
--dry-run Show what would be done without executing
|
||||
--help, -h Show this help message
|
||||
`);
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
return args;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// UTILITIES
|
||||
// ============================================================================
|
||||
|
||||
function execCapture({ cmd }: { cmd: string }): string {
|
||||
try {
|
||||
return execSync(cmd, { encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim();
|
||||
} catch (error: any) {
|
||||
return error.stdout?.trim() ?? "";
|
||||
}
|
||||
}
|
||||
|
||||
function tbSql({ query }: { query: string }): string {
|
||||
const escaped = query.replace(/"/g, '\\"');
|
||||
try {
|
||||
const result = execSync(`tb --cloud sql "${escaped}"`, {
|
||||
encoding: "utf-8",
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
return result.trim();
|
||||
} catch (error: any) {
|
||||
return error.stdout?.trim() ?? "";
|
||||
}
|
||||
}
|
||||
|
||||
function exec({ cmd }: { cmd: string }): void {
|
||||
execSync(cmd, { encoding: "utf-8", stdio: "inherit" });
|
||||
}
|
||||
|
||||
async function sleep({ ms }: { ms: number }): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function extractNumber({ result }: { result: string }): number | null {
|
||||
for (const line of result.split("\n")) {
|
||||
const cleaned = line.trim();
|
||||
const num = Number(cleaned);
|
||||
if (!isNaN(num) && cleaned !== "" && String(Math.round(num)) === cleaned) {
|
||||
return num;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function formatEpochMs({ epochMs }: { epochMs: number }): string {
|
||||
return new Date(epochMs).toISOString();
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// TINYBIRD QUERIES
|
||||
// ============================================================================
|
||||
|
||||
function getMaxCreatedAtInTinybird(): number | null {
|
||||
console.log("Querying Tinybird for backfill progress...");
|
||||
const result = tbSql({
|
||||
query: "SELECT max(created_at) FROM organizations WHERE __action = 'read'",
|
||||
});
|
||||
const num = extractNumber({ result });
|
||||
if (num && num > 0) {
|
||||
console.log(` Tinybird MAX created_at (backfill rows): ${num} (${formatEpochMs({ epochMs: num })})`);
|
||||
return num;
|
||||
}
|
||||
console.log(" Tinybird MAX created_at: No backfilled rows yet");
|
||||
return null;
|
||||
}
|
||||
|
||||
function getOrganizationCount(): number {
|
||||
const result = tbSql({ query: "SELECT count() FROM organizations FINAL WHERE __action != 'delete'" });
|
||||
return extractNumber({ result }) ?? 0;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// CHUNK GENERATION (FORWARD)
|
||||
// ============================================================================
|
||||
|
||||
function generateChunksForward({
|
||||
startEpochMs,
|
||||
endEpochMs,
|
||||
chunkDays,
|
||||
}: {
|
||||
startEpochMs: number;
|
||||
endEpochMs: number;
|
||||
chunkDays: number;
|
||||
}): Chunk[] {
|
||||
const chunks: Chunk[] = [];
|
||||
const chunkMs = chunkDays * 24 * 60 * 60 * 1000;
|
||||
let current = startEpochMs;
|
||||
|
||||
while (current < endEpochMs) {
|
||||
const next = Math.min(current + chunkMs, endEpochMs);
|
||||
chunks.push({ startEpochMs: current, endEpochMs: next });
|
||||
current = next;
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// COPY JOB EXECUTION
|
||||
// ============================================================================
|
||||
|
||||
async function waitForCopyJobs(): Promise<void> {
|
||||
const maxAttempts = 60;
|
||||
|
||||
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
||||
const waiting = execCapture({
|
||||
cmd: "tb --cloud job ls --status waiting --kind copy 2>/dev/null | grep -c '^id:' || echo 0",
|
||||
});
|
||||
const working = execCapture({
|
||||
cmd: "tb --cloud job ls --status working --kind copy 2>/dev/null | grep -c '^id:' || echo 0",
|
||||
});
|
||||
const total = parseInt(waiting, 10) + parseInt(working, 10);
|
||||
|
||||
if (total === 0) return;
|
||||
if (attempt === 0) console.log(` Waiting for ${total} existing copy job(s) to complete...`);
|
||||
|
||||
await sleep({ ms: 5000 });
|
||||
}
|
||||
|
||||
throw new Error("Timed out waiting for existing copy jobs to complete");
|
||||
}
|
||||
|
||||
async function runCopyJob({
|
||||
startEpochMs,
|
||||
endEpochMs,
|
||||
retries = MAX_RETRIES,
|
||||
}: {
|
||||
startEpochMs: number;
|
||||
endEpochMs: number;
|
||||
retries?: number;
|
||||
}): Promise<boolean> {
|
||||
for (let attempt = 1; attempt <= retries; attempt++) {
|
||||
try {
|
||||
await waitForCopyJobs();
|
||||
exec({
|
||||
cmd: `tb --cloud copy run organizations_backfill --param start_epoch_ms="${startEpochMs}" --param end_epoch_ms="${endEpochMs}" --wait`,
|
||||
});
|
||||
return true;
|
||||
} catch (error: any) {
|
||||
const msg = error.message || error.toString();
|
||||
if (attempt < retries) {
|
||||
console.log(` Attempt ${attempt}/${retries} failed. Retrying in ${RETRY_DELAY_MS / 1000}s...`);
|
||||
console.log(` Error: ${msg.substring(0, 200)}`);
|
||||
await sleep({ ms: RETRY_DELAY_MS });
|
||||
} else {
|
||||
console.error(` All ${retries} attempts failed for chunk ${startEpochMs} -> ${endEpochMs}`);
|
||||
console.error(` Error: ${msg}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// MAIN
|
||||
// ============================================================================
|
||||
|
||||
async function main() {
|
||||
console.log("=== Organizations Backfill Script ===\n");
|
||||
|
||||
const args = parseArgs();
|
||||
|
||||
// 1. Find resume point from Tinybird
|
||||
const tinybirdMax = getMaxCreatedAtInTinybird();
|
||||
|
||||
// Resume from just after the last backfilled row, or from the very beginning
|
||||
const resumePoint = tinybirdMax ? tinybirdMax + 1 : START_EPOCH_MS;
|
||||
const endPoint = END_EPOCH_MS;
|
||||
|
||||
console.log(`\nResume point: ${resumePoint} (${formatEpochMs({ epochMs: resumePoint })})`);
|
||||
console.log(`End point: ${endPoint} (${formatEpochMs({ epochMs: endPoint })})`);
|
||||
|
||||
// 2. Check if done
|
||||
if (resumePoint >= endPoint) {
|
||||
console.log("\n✓ Backfill complete! Nothing more to do.");
|
||||
console.log(` Total organizations in Tinybird: ${getOrganizationCount()}`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// 3. Generate chunks
|
||||
const chunks = generateChunksForward({
|
||||
startEpochMs: resumePoint,
|
||||
endEpochMs: endPoint,
|
||||
chunkDays: args.chunkDays,
|
||||
});
|
||||
|
||||
console.log(`\nGenerated ${chunks.length} chunks (${args.chunkDays} days each)`);
|
||||
console.log(`Direction: oldest → newest\n`);
|
||||
|
||||
// 4. Dry run
|
||||
if (args.dryRun) {
|
||||
console.log("DRY RUN - Would process these chunks:\n");
|
||||
chunks.forEach((chunk, i) => {
|
||||
console.log(
|
||||
` ${i + 1}. ${formatEpochMs({ epochMs: chunk.startEpochMs })} -> ${formatEpochMs({ epochMs: chunk.endEpochMs })}`,
|
||||
);
|
||||
});
|
||||
console.log(`\nTotal: ${chunks.length} chunks`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// 5. Process chunks
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const chunk = chunks[i];
|
||||
const chunkNum = i + 1;
|
||||
|
||||
console.log(
|
||||
`=== Chunk ${chunkNum}/${chunks.length}: ${formatEpochMs({ epochMs: chunk.startEpochMs })} -> ${formatEpochMs({ epochMs: chunk.endEpochMs })} ===`,
|
||||
);
|
||||
|
||||
const startTime = Date.now();
|
||||
const success = await runCopyJob({
|
||||
startEpochMs: chunk.startEpochMs,
|
||||
endEpochMs: chunk.endEpochMs,
|
||||
});
|
||||
const duration = Math.round((Date.now() - startTime) / 1000);
|
||||
|
||||
if (success) {
|
||||
const count = getOrganizationCount();
|
||||
console.log(`✓ Chunk ${chunkNum} COMPLETE in ${duration}s. Organizations in Tinybird: ${count}\n`);
|
||||
} else {
|
||||
console.log(`✗ Chunk ${chunkNum} FAILED after ${duration}s\n`);
|
||||
console.log("To resume, simply re-run:");
|
||||
console.log(" bun scripts/backfill_organizations.ts\n");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (i < chunks.length - 1) {
|
||||
await sleep({ ms: DELAY_BETWEEN_CHUNKS_MS });
|
||||
}
|
||||
}
|
||||
|
||||
// 6. Summary
|
||||
console.log("==========================================");
|
||||
console.log("=== Backfill Complete ===");
|
||||
console.log(`Total organizations in Tinybird: ${getOrganizationCount()}`);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error("Fatal error:", error);
|
||||
process.exit(1);
|
||||
});
|
||||
170
server/tinybird/scripts/backfill_rollovers.ts
Normal file
170
server/tinybird/scripts/backfill_rollovers.ts
Normal file
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env bun
|
||||
|
||||
/**
|
||||
* Offset-based idempotent backfill for rollovers from Postgres to Tinybird.
|
||||
*
|
||||
* Rollovers have no created_at so we can't chunk by date.
|
||||
* Pages through the full table with LIMIT + OFFSET ordered by id.
|
||||
* Stops when a page returns fewer rows than page_size.
|
||||
*
|
||||
* Usage:
|
||||
* bun scripts/backfill_rollovers.ts
|
||||
* bun scripts/backfill_rollovers.ts --page-size 1000
|
||||
* bun scripts/backfill_rollovers.ts --offset 10000
|
||||
* bun scripts/backfill_rollovers.ts --dry-run
|
||||
*/
|
||||
|
||||
import {
|
||||
MAX_RETRIES,
|
||||
RETRY_DELAY_MS,
|
||||
DELAY_BETWEEN_CHUNKS_MS,
|
||||
exec,
|
||||
tbSql,
|
||||
sleep,
|
||||
extractNumber,
|
||||
waitForCopyJobs,
|
||||
} from "./backfill_base.js";
|
||||
|
||||
// ============================================================================
|
||||
// CONFIGURATION
|
||||
// ============================================================================
|
||||
|
||||
const DATASOURCE = "rollovers";
|
||||
const COPY_PIPE = "rollovers_backfill";
|
||||
const DEFAULT_PAGE_SIZE = 5000;
|
||||
|
||||
// ============================================================================
|
||||
// ARGUMENT PARSING
|
||||
// ============================================================================
|
||||
|
||||
function parseArgs(): { pageSize: number; offsetOverride?: number; dryRun: boolean } {
|
||||
const args = { pageSize: DEFAULT_PAGE_SIZE, offsetOverride: undefined as number | undefined, dryRun: false };
|
||||
|
||||
for (let i = 2; i < process.argv.length; i++) {
|
||||
const arg = process.argv[i];
|
||||
if (arg === "--page-size" && process.argv[i + 1]) {
|
||||
args.pageSize = parseInt(process.argv[++i], 10);
|
||||
} else if (arg === "--offset" && process.argv[i + 1]) {
|
||||
args.offsetOverride = parseInt(process.argv[++i], 10);
|
||||
} else if (arg === "--dry-run") {
|
||||
args.dryRun = true;
|
||||
} else if (arg === "--help" || arg === "-h") {
|
||||
console.log(`
|
||||
Offset-based backfill for rollovers
|
||||
|
||||
Usage: bun scripts/backfill_rollovers.ts [options]
|
||||
|
||||
Options:
|
||||
--page-size <n> Rows per page (default: ${DEFAULT_PAGE_SIZE})
|
||||
--offset <n> Start from this offset (default: 0)
|
||||
--dry-run Print plan without executing
|
||||
--help, -h Show this help
|
||||
`);
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
return args;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// HELPERS
|
||||
// ============================================================================
|
||||
|
||||
async function getRowCount(): Promise<number> {
|
||||
const rows = await tbSql({
|
||||
query: `SELECT count() AS val FROM ${DATASOURCE} FINAL WHERE __action != 'delete'`,
|
||||
});
|
||||
return extractNumber({ rows, col: "val" }) ?? 0;
|
||||
}
|
||||
|
||||
async function runPage({
|
||||
offset,
|
||||
pageSize,
|
||||
retries = MAX_RETRIES,
|
||||
}: {
|
||||
offset: number;
|
||||
pageSize: number;
|
||||
retries?: number;
|
||||
}): Promise<boolean> {
|
||||
for (let attempt = 1; attempt <= retries; attempt++) {
|
||||
try {
|
||||
await waitForCopyJobs();
|
||||
exec({
|
||||
cmd: `TB_VERSION_WARNING=0 tb --cloud copy run ${COPY_PIPE} --param offset="${offset}" --param page_size="${pageSize}" --wait`,
|
||||
});
|
||||
return true;
|
||||
} catch (error: any) {
|
||||
const msg = error.message || error.toString();
|
||||
if (attempt < retries) {
|
||||
console.log(` Attempt ${attempt}/${retries} failed. Retrying in ${RETRY_DELAY_MS / 1000}s...`);
|
||||
console.log(` Error: ${msg.substring(0, 200)}`);
|
||||
await sleep({ ms: RETRY_DELAY_MS });
|
||||
} else {
|
||||
console.error(` All ${retries} attempts failed at offset ${offset}`);
|
||||
console.error(` Error: ${msg}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// MAIN
|
||||
// ============================================================================
|
||||
|
||||
async function main() {
|
||||
console.log("=== Rollovers Backfill (Offset-based) ===\n");
|
||||
|
||||
const args = parseArgs();
|
||||
let offset = args.offsetOverride ?? 0;
|
||||
|
||||
console.log(`Page size: ${args.pageSize}`);
|
||||
console.log(`Starting offset: ${offset}\n`);
|
||||
|
||||
if (args.dryRun) {
|
||||
console.log("DRY RUN — would run pages starting at offset", offset);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
let pageNum = 0;
|
||||
|
||||
while (true) {
|
||||
pageNum++;
|
||||
console.log(`=== Page ${pageNum}: offset=${offset} ===`);
|
||||
const startTime = Date.now();
|
||||
|
||||
const success = await runPage({ offset, pageSize: args.pageSize });
|
||||
const duration = Math.round((Date.now() - startTime) / 1000);
|
||||
|
||||
if (!success) {
|
||||
console.log(`✗ Page ${pageNum} FAILED after ${duration}s`);
|
||||
console.log("To resume from this point, re-run with:");
|
||||
console.log(` bun scripts/backfill_rollovers.ts --offset ${offset}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const rowCount = await getRowCount();
|
||||
console.log(`✓ Page ${pageNum} done in ${duration}s. Rows in Tinybird: ${rowCount}\n`);
|
||||
|
||||
// A page returning fewer rows than page_size means we've hit the end
|
||||
// We detect this by comparing expected next offset vs actual row count
|
||||
offset += args.pageSize;
|
||||
if (rowCount < offset) {
|
||||
console.log("Last page was partial — backfill complete.");
|
||||
break;
|
||||
}
|
||||
|
||||
await sleep({ ms: DELAY_BETWEEN_CHUNKS_MS });
|
||||
}
|
||||
|
||||
console.log("==========================================");
|
||||
console.log("=== Rollovers Backfill Complete ===");
|
||||
console.log(`Total rollovers in Tinybird: ${await getRowCount()}`);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error("Fatal error:", error);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user