fix: run migration task

This commit is contained in:
John Yeo
2025-11-18 20:22:16 +00:00
parent ac24473e6d
commit 36c40d1622
8 changed files with 679 additions and 86 deletions

View File

@@ -0,0 +1,446 @@
# Express to Hono Migration Tracker
This document tracks all Express endpoints in the codebase and their migration status to Hono.
## ✅ Already Migrated to Hono
Based on `initHono.ts`, the following routes are already using Hono:
### API Routes (v1)
- `/v1/balances/*` - `balancesRouter` (Hono)
- `/v1/migrations` - `migrationRouter` (Hono)
- `/v1/entities/*` - `entityRouter` (Hono)
- `/v1/customers/*` - `cusRouter` (Hono)
- `/v1/products_beta/*` - `honoProductBetaRouter` (Hono)
- `/v1/products/*` - `honoProductRouter` (Hono)
- `/v1/plans/*` - `honoProductRouter` (Hono)
- `/v1/features/*` - `featureRouter` (Hono)
- `/v1/platform/*` - `platformBetaRouter` (Hono)
- `/v1/platform/beta/*` - `platformBetaRouter` (Hono)
- `/v1/organization/*` - `honoOrgRouter` (Hono)
- `/v1/billing/*` - `billingRouter` (Hono)
### Internal/Dashboard Routes
- `/products/*` - `internalProductRouter` (Hono) - With betterAuthMiddleware
- `/customers/*` - `internalCusRouter` (Hono) - With betterAuthMiddleware
### Webhooks
- `/webhooks/connect/:env` - `handleConnectWebhook` (Hono)
- `/webhooks/vercel/*` - `vercelWebhookRouter` (Hono)
### Other
- `/stripe/oauth_callback` - `handleOAuthCallback` (Hono)
- `/` - Health check (Hono)
---
## 🚨 Still on Express - Needs Migration
### 1. Main Router (`/` - mainRouter.ts)
**Express Endpoints:**
```
GET / # Hello World endpoint
POST /organization # handlePostOrg (withAuth)
```
**Sub-routers:**
- `/admin` - adminRouter (withAdminAuth)
- `/users` - userRouter (withAuth)
- `/onboarding` - onboardingRouter (withOrgAuth)
- `/organization` - orgRouter (withOrgAuth)
- `/products` - expressProductRouter (withOrgAuth)
- `/dev` - devRouter
- `/customers` - cusRouter (withOrgAuth)
- `/query` - analyticsRouter (withOrgAuth)
- `/saved_views` - viewsRouter (withOrgAuth)
- `/trmnl` - trmnlRouter
**Special Routes:**
```
GET /invoices/hosted_invoice_url/:invoiceId # Rate limited (10/min)
POST /api/autumn # Autumn SDK handler (withOrgAuth)
POST /demo/api/autumn # Demo Autumn handler (withOrgAuth)
```
---
### 2. Admin Router (`/admin` - adminRouter.ts)
**Middleware:** withAdminAuth
**Endpoints:**
```
GET /admin/users # Search users with pagination
GET /admin/orgs # Search orgs with pagination
```
---
### 3. User Router (`/users` - userRouter.ts)
**Middleware:** withAuth
**Endpoints:**
```
GET /users # Get current user info
```
---
### 4. Onboarding Router (`/onboarding` - onboardingRouter.ts)
**Middleware:** withOrgAuth
**Endpoints:**
```
POST /onboarding # Create products/features from chat result
```
---
### 5. Organization Router (`/organization` - orgRouter.ts)
**Middleware:** withOrgAuth
**Express Endpoints (NOT migrated):**
```
GET /organization/members # handleGetOrgMembers
POST /organization/remove-member # handleRemoveMember
GET /organization/upload_url # handleGetUploadUrl
GET /organization/invites # handleGetInvites
DELETE /organization # handleDeleteOrg
DELETE /organization/delete-user # Delete user (returns success)
GET /organization # handleGetOrg
```
**Note:** This router has both Express and Hono versions. The Hono version (`honoOrgRouter`) has:
- PATCH `/organization` - handleUpdateOrg
- GET `/organization/stripe` - handleGetStripeAccount
- DELETE `/organization/stripe` - handleDeleteStripe
- POST `/organization/stripe` - handleConnectStripe
- GET `/organization/stripe/oauth_url` - handleGetOAuthUrl
- POST `/organization/reset_default_account` - handleResetDefaultAccount
- PATCH `/organization/vercel` - handleUpsertVercelConfig
- GET `/organization/vercel_sink` - handleGetVercelSink
---
### 6. Products Router (Internal) (`/products` - internalProductRouter.ts)
**Middleware:** withOrgAuth
**Express Endpoints:**
```
GET /products/products # List all products
GET /products/product_counts # Get counts for all products
GET /products/features # Get all features
GET /products/rewards # Get rewards and reward programs
GET /products/migrations # Get list of migrations
GET /products/data # Get all product data (GET)
POST /products/data # Get all product data with filters (POST)
POST /products/product_options # Get product options
GET /products/:productId/info # handleGetProductDeleteInfo
GET /products/rewards # Get rewards
GET /products/has_entity_feature_id # Check if has entity feature ID
GET /products/counts # Get product counts with filters
```
**Hono Endpoints (already migrated):**
```
GET /products/:productId/count # handleGetProductCount
GET /products/:productId/data # handleGetProductInternal
POST /products/copy_to_production # handleCopyEnvironment
```
---
### 7. Dev Router (`/dev` - devRouter.ts)
**Endpoints:**
```
GET /dev/data # Get API keys, org, svix dashboard URL (withOrgAuth)
POST /dev/api_key # Create API key (withOrgAuth)
DELETE /dev/api_key/:id # Delete API key (withOrgAuth)
POST /dev/otp # handleCreateOtp (withOrgAuth)
GET /dev/otp/:otp # handleGetOtp
POST /dev/cli/stripe # Update Stripe keys from CLI
```
---
### 8. Customers Router (Internal) (`/customers` - internalCusRouter.ts)
**Middleware:** withOrgAuth
**Express Endpoints:**
```
POST /customers/all/search # Search customers
GET /customers/:customer_id/events # Get customer events
POST /customers/all/full_customers # Get full customers
GET /customers/:customer_id/product/:product_id # Get customer product
```
**Hono Endpoints (already migrated):**
```
GET /customers/:customer_id # handleGetCustomerInternal
GET /customers/:customer_id/referrals # handleGetCusReferrals
```
---
### 9. Analytics Router (`/query` - analyticsRouter.ts)
**Middleware:** withOrgAuth
**Endpoints:**
```
GET /query/event_names # Get top event names
POST /query/events # Query events by customer ID
POST /query/raw # Query raw events by customer ID
```
---
### 10. Saved Views Router (`/saved_views` - savedViewsRouter.ts)
**Middleware:** withOrgAuth
**Endpoints:**
```
POST /saved_views/save # ViewsService.saveView
GET /saved_views # ViewsService.getViews
DELETE /saved_views/:viewId # ViewsService.deleteView
```
---
### 11. TRMNL Router (`/trmnl` - trmnlRouter.ts)
**Endpoints:**
```
GET /trmnl/device_id # Get TRMNL config (withOrgAuth)
POST /trmnl/device_id # Save TRMNL device ID (withOrgAuth)
POST /trmnl/screen # Generate TRMNL screen (trmnlLimiter + trmnlAuthMiddleware)
```
---
### 12. API Router (`/v1` - apiRouter.ts)
**Middleware Chain:**
- apiAuthMiddleware
- pricingMiddleware
- analyticsMiddleware
- expressApiVersionMiddleware
- refreshCacheMiddleware
**Sub-routers still on Express:**
```
/v1/invoices # invoiceRouter
/v1/components # componentRouter
/v1/rewards # rewardRouter
/v1/reward_programs # rewardProgramRouter
/v1/referrals # referralRouter
/v1/redemptions # redemptionRouter
/v1 # attachRouter
/v1/cancel # cancelRouter
/v1/query # analyticsRouter (duplicate of /query)
/v1/platform # platformRouter
/v1/products # expressProductRouter
/v1/customers # expressCusRouter
```
**Endpoints:**
```
GET /v1/organization # handleGetOrg
```
---
### 13. Invoice Router (`/v1/invoices` - invoiceRouter.ts)
**Endpoints:**
```
GET /v1/invoices/:stripe_invoice_id/stripe # Get Stripe invoice
```
---
### 14. Component Router (`/v1/components` - componentRouter.ts)
**Endpoints:**
```
GET /v1/components/pricing_table # Get pricing table
```
---
### 15. Reward Router (`/v1/rewards` - rewardRouter.ts)
**Endpoints:**
```
POST /v1/rewards # handleCreateCoupon
DELETE /v1/rewards/:id # handleDeleteCoupon
POST /v1/rewards/:internalId # handleUpdateCoupon
GET /v1/rewards/:id # handleGetCoupon
```
---
### 16. Reward Program Router (`/v1/reward_programs` - rewardProgramRouter.ts)
**Endpoints:**
```
POST /v1/reward_programs # handleCreateRewardProgram
DELETE /v1/reward_programs/:id # handleDeleteRewardProgram
PUT /v1/reward_programs/:id # Update reward program
```
---
### 17. Referral Router (`/v1/referrals` - referralRouter.ts)
**Endpoints:**
```
POST /v1/referrals/code # handleGetReferralCode
POST /v1/referrals/redeem # handleRedeemReferral
```
---
### 18. Redemption Router (`/v1/redemptions` - redemptionRouter.ts)
**Endpoints:**
```
GET /v1/redemptions/:redemptionId # handleGetRedemption
```
---
### 19. Attach Router (`/v1` - attachRouter.ts)
**Endpoints:**
```
POST /v1/attach # handleAttach
POST /v1/attach/preview # handleAttachPreview
POST /v1/checkout # handleCheckout
```
---
### 20. Cancel Router (`/v1/cancel` - cancelRouter.ts)
**Endpoints:**
```
POST /v1/cancel # Cancel/expire customer product
```
---
### 21. Platform Router (`/v1/platform` - platformRouter.ts)
**Middleware:** platformAuthMiddleware
**Endpoints:**
```
POST /v1/platform/exchange # Exchange platform credentials
```
**Note:** There's also a `/v1/platform/beta` route that's already migrated to Hono
---
### 22. Products Router (API) (`/v1/products` - productRouter.ts)
**Express Endpoints:**
```
GET /v1/products/:product_id/has_customers # handlePlanHasCustomers
```
**Note:** Most product routes are already migrated to Hono via `honoProductRouter`
---
### 23. Customers Router (API) (`/v1/customers` - cusRouter.ts)
**Express Endpoints:**
```
GET /v1/customers/:customer_id/billing_portal # handleGetBillingPortal
```
**Note:** Most customer routes are already migrated to Hono via `cusRouter` (Hono)
---
### 24. Webhooks Router (`/webhooks` - webhooksRouter.ts)
**Endpoints:**
```
POST /webhooks/stripe/:orgId/:env # stripeWebhookRouter
POST /webhooks/autumn # autumnWebhookRouter
```
**Note:** Vercel webhooks are already migrated to Hono
---
## Migration Priority
### High Priority (Core API)
1.`/v1/attach` - attachRouter
2.`/v1/cancel` - cancelRouter
3. `/v1/invoices` - invoiceRouter
4. `/v1/components` - componentRouter
### Medium Priority (Features)
5. `/v1/rewards` - rewardRouter
6. `/v1/reward_programs` - rewardProgramRouter
7. `/v1/referrals` - referralRouter
8. `/v1/redemptions` - redemptionRouter
9. `/v1/platform` - platformRouter (legacy)
### Lower Priority (Internal/Dashboard)
10. `/organization` - orgRouter (Express parts)
11. `/products` - expressProductRouter (Express parts)
12. `/customers` - cusRouter (Express parts)
13. `/query` - analyticsRouter
14. `/saved_views` - viewsRouter
15. `/dev` - devRouter
16. `/trmnl` - trmnlRouter
### Admin/System
17. `/admin` - adminRouter
18. `/users` - userRouter
19. `/onboarding` - onboardingRouter
20. Main router special routes (Autumn SDK handlers, invoice URL)
### Webhooks
21. `/webhooks/stripe` - stripeWebhookRouter
22. `/webhooks/autumn` - autumnWebhookRouter
---
## Notes
- **Authentication patterns differ:** Express uses `withAuth`, `withOrgAuth`, `withAdminAuth` middleware, while Hono uses `betterAuthMiddleware` and `secretKeyMiddleware`
- **Some routers are hybrid:** They have both Express and Hono implementations (e.g., orgRouter, productRouter, cusRouter)
- **Middleware migration:** Each Express router has specific middleware that needs to be converted to Hono middleware equivalents
- **Rate limiting:** Some routes use `express-rate-limit` which needs Hono equivalents
- **Special handlers:** Routes like Autumn SDK integration and invoice redirects need careful consideration
---
## Checklist for Each Migration
When migrating an Express route to Hono:
- [ ] Convert route handler from Express format to Hono format
- [ ] Migrate middleware (auth, validation, etc.)
- [ ] Update error handling (use `RecaseError` consistently)
- [ ] Test authentication flow
- [ ] Update `initHono.ts` to include the new route
- [ ] Remove or mark deprecated in Express router
- [ ] Update tests
- [ ] Update API documentation

View File

@@ -0,0 +1,197 @@
# Express to Hono Migration Summary
## Quick Stats
### ✅ Migrated (Hono)
- **12 major route groups** fully migrated
- **~80+ endpoints** on Hono
### 🚨 Remaining (Express)
- **24 route groups** still on Express
- **~100+ endpoints** to migrate
---
## What's Already Migrated ✅
### Core API (v1)
- Balances
- Migrations
- Entities
- Customers (most routes)
- Products/Plans (most routes)
- Features
- Platform Beta
- Organization (partial - CRUD operations)
- Billing
### Internal/Dashboard
- Products (detail/count routes)
- Customers (detail/referrals routes)
### Webhooks
- Vercel webhooks
- Connect webhooks
---
## What Still Needs Migration 🚨
### Critical API Routes
```
/v1/attach # Attach products to customers
/v1/attach/preview # Preview attachment
/v1/checkout # Checkout flow
/v1/cancel # Cancel/expire products
/v1/invoices # Invoice operations
/v1/components # Pricing table, etc.
```
### Rewards & Referrals
```
/v1/rewards # CRUD for rewards/coupons
/v1/reward_programs # CRUD for reward programs
/v1/referrals # Get referral codes, redeem
/v1/redemptions # Get redemption info
```
### Platform & Analytics
```
/v1/platform # Platform API (legacy)
/query # Analytics queries
```
### Internal/Dashboard
```
/organization # Org management (partial)
/products # Product management (partial)
/customers # Customer management (partial)
/dev # API keys, OTP, CLI
/saved_views # Saved view CRUD
/trmnl # TRMNL integration
/admin # Admin operations
/users # User management
/onboarding # Onboarding flow
```
### Webhooks
```
/webhooks/stripe # Stripe webhooks
/webhooks/autumn # Autumn webhooks
```
### Special Routes
```
GET /invoices/hosted_invoice_url/:invoiceId # Invoice redirects
POST /api/autumn # Autumn SDK handler
POST /demo/api/autumn # Demo Autumn handler
```
---
## Migration Strategy
### Phase 1: Critical API Routes (Priority 1)
Target: **Core billing & product operations**
- [ ] `/v1/attach` endpoints
- [ ] `/v1/cancel` endpoint
- [ ] `/v1/invoices` endpoints
- [ ] `/v1/components` endpoints
### Phase 2: Features & Extensions (Priority 2)
Target: **Rewards, referrals, platform**
- [ ] `/v1/rewards` endpoints
- [ ] `/v1/reward_programs` endpoints
- [ ] `/v1/referrals` endpoints
- [ ] `/v1/redemptions` endpoints
- [ ] `/v1/platform` (legacy) endpoints
### Phase 3: Internal/Dashboard (Priority 3)
Target: **Dashboard functionality**
- [ ] Complete `/organization` migration
- [ ] Complete `/products` migration
- [ ] Complete `/customers` migration
- [ ] `/query` analytics endpoints
- [ ] `/saved_views` endpoints
- [ ] `/dev` endpoints
### Phase 4: System & Admin (Priority 4)
Target: **Admin, auth, system routes**
- [ ] `/admin` endpoints
- [ ] `/users` endpoints
- [ ] `/onboarding` endpoints
- [ ] `/trmnl` endpoints
- [ ] Special routes (Autumn SDK handlers, etc.)
### Phase 5: Webhooks (Priority 5)
Target: **External webhook handlers**
- [ ] `/webhooks/stripe`
- [ ] `/webhooks/autumn`
---
## Key Migration Challenges
### 1. Authentication Differences
- **Express:** `withAuth`, `withOrgAuth`, `withAdminAuth` middleware
- **Hono:** `betterAuthMiddleware`, `secretKeyMiddleware`
- Need to ensure consistent auth behavior
### 2. Middleware Migration
Each Express router has its own middleware stack:
- `apiAuthMiddleware`
- `pricingMiddleware`
- `analyticsMiddleware`
- `expressApiVersionMiddleware`
- `refreshCacheMiddleware`
These need Hono equivalents (most already exist in `honoMiddlewares/`)
### 3. Rate Limiting
- Express uses `express-rate-limit`
- Need Hono rate limiting solution
### 4. Raw Body Handling
- Webhook routes use `express.raw({ type: "application/json" })`
- Need to handle raw body in Hono for webhook signature verification
### 5. Special Integrations
- Autumn SDK handler (uses `autumnHandler` from `autumn-js/express`)
- Need to find/create Hono equivalent
---
## Testing Strategy
For each migrated route:
1. ✅ Unit tests for handler logic
2. ✅ Integration tests for full request flow
3. ✅ Authentication/authorization tests
4. ✅ Error handling tests
5. ✅ Backward compatibility (if needed)
---
## Resources
- **Full detailed list:** See `EXPRESS_TO_HONO_MIGRATION.md`
- **Hono middleware:** `/server/src/honoMiddlewares/`
- **Hono routers:** Various `*Router.ts` files using `new Hono<HonoEnv>()`
- **Init file:** `/server/src/initHono.ts`
---
## Progress Tracking
Last updated: 2024-11-18
| Phase | Status | Progress |
|-------|--------|----------|
| Phase 1: Critical API | 🚧 In Progress | 0% |
| Phase 2: Features | ⏳ Not Started | 0% |
| Phase 3: Internal | 🚧 Partial | 30% |
| Phase 4: System | ⏳ Not Started | 0% |
| Phase 5: Webhooks | ⏳ Not Started | 0% |
**Overall Migration:** ~45% complete (estimated)

View File

@@ -41,15 +41,16 @@ export const runSyncBalanceBatch = async ({
logger.info(`🔄 Starting sync: ${itemDescription}`);
try {
await syncItem({ item, ctx });
logger.info(`✅ Successfully synced: ${itemDescription}`);
} catch (error) {
logger.error(`❌ Failed to sync: ${itemDescription}`, {
error: error instanceof Error ? error : new Error(String(error)),
item,
});
// Re-throw to trigger SQS retry
throw error;
}
await syncItem({ item, ctx });
logger.info(`✅ Successfully synced: ${itemDescription}`);
// try {
// } catch (error) {
// logger.error(`❌ Failed to sync: ${itemDescription}`, {
// error: error instanceof Error ? error : new Error(String(error)),
// item,
// });
// // Re-throw to trigger SQS retry
// throw error;
// }
};

View File

@@ -13,7 +13,6 @@ import { trmnlRouter } from "./api/trmnl/trmnlRouter.js";
import { cusRouter } from "./customers/internalCusRouter.js";
import { devRouter } from "./dev/devRouter.js";
import { InvoiceService } from "./invoices/InvoiceService.js";
import { handlePostOrg } from "./orgs/handlers/handlePostOrg.js";
import { onboardingRouter } from "./orgs/onboarding/onboardingRouter.js";
import { orgRouter } from "./orgs/orgRouter.js";
import { expressProductRouter } from "./products/internalProductRouter.js";
@@ -26,7 +25,6 @@ mainRouter.get("", async (req: any, res) => {
res.status(200).json({ message: "Hello World" });
});
mainRouter.post("/organization", withAuth, handlePostOrg);
mainRouter.use("/admin", withAdminAuth, adminRouter);
mainRouter.use("/users", withAuth, userRouter);
mainRouter.use("/onboarding", withOrgAuth, onboardingRouter);

View File

@@ -1,7 +1,6 @@
/** biome-ignore-all lint/suspicious/noExplicitAny: ok */
import { MigrationJobStep } from "@autumn/shared";
import type { DrizzleCli } from "@/db/initDrizzle.js";
import { timeout } from "../../utils/genUtils.js";
import { FeatureService } from "../features/FeatureService.js";
import { ProductService } from "../products/ProductService.js";
import { MigrationService } from "./MigrationService.js";
@@ -42,8 +41,6 @@ export const runMigrationTask = async ({
},
});
await timeout(10000);
const { org_id: orgId, env } = migrationJob;
// Get from and to products

View File

@@ -1,62 +0,0 @@
import type {
ExtendedRequest,
ExtendedResponse,
} from "@/utils/models/Request.js";
import { routeHandler } from "@/utils/routerUtils.js";
export const handlePostOrg = async (req: any, res: any) =>
routeHandler({
req,
res,
action: "create org",
handler: async (req: ExtendedRequest, res: ExtendedResponse) => {
const { userId, db } = req;
console.log("userId", userId);
// const userMemberships = await auth.api.({
// userId: userId!,
// });
// const clerk = createClerkCli();
// const user = await clerk.users.getUser(userId!);
// let userMemberships = await clerk.users.getOrganizationMembershipList({
// userId: userId!,
// });
// let org;
// if (userMemberships.data.length === 0) {
// org = await clerk.organizations.createOrganization({
// name: `${user.firstName}'s Org`,
// });
// // 2. Create org membership for user
// await clerk.organizations.createOrganizationMembership({
// organizationId: org.id,
// userId: userId!,
// role: "org:admin",
// });
// await saveOrgToDB({
// db,
// id: org.id,
// slug: org.slug,
// });
// console.log(`Created new org: ${org.id} (${org.slug})`);
// } else {
// org = userMemberships.data[0].organization;
// }
res.status(200).json({
id: "123",
slug: "123",
});
// res.status(200).json({
// id: org.id,
// slug: org.slug,
// });
},
});

View File

@@ -3,6 +3,7 @@ import {
ErrCode,
MigrateProductParamsSchema,
ProductNotFoundError,
RecaseError,
type UsagePriceConfig,
} from "@autumn/shared";
import { createRoute } from "@/honoMiddlewares/routeHandler.js";
@@ -16,7 +17,6 @@ import {
import { isFreeProduct } from "@/internal/products/productUtils.js";
import { JobName } from "@/queue/JobName.js";
import { addTaskToQueue } from "@/queue/queueUtils.js";
import RecaseError from "@/utils/errorUtils.js";
import { findPrepaidPrice } from "../prices/priceUtils/findPriceUtils.js";
/**
@@ -48,6 +48,18 @@ export const handleMigrateProductV2 = createRoute({
version: to_version,
});
const currentMigrations = await MigrationService.getExistingJobs({
db,
orgId: org.id,
env,
});
if (currentMigrations.length > 0) {
throw new RecaseError({
message: "Another migration is ongoing, cannot create a new migration",
});
}
if (!fromProduct || !toProduct) {
throw new ProductNotFoundError({
productId: !fromProduct ? from_product_id : to_product_id,
@@ -124,6 +136,7 @@ export const handleMigrateProductV2 = createRoute({
});
// Add task to queue for processing
await addTaskToQueue({
jobName: JobName.Migration,
payload: {

View File

@@ -139,13 +139,16 @@ const processMessage = async ({
});
}
} catch (error: any) {
workerLogger.error(`Failed to process SQS job: ${job.name}`, {
jobName: job.name,
error: {
message: error.message,
stack: error.stack,
workerLogger.error(
`Failed to process SQS job: ${job.name}, error: ${error}`,
{
jobName: job.name,
error: {
message: error.message,
stack: error.stack,
},
},
});
);
// Don't delete the message on error - it will become visible again for retry
throw error;
}
@@ -168,7 +171,7 @@ const startPollingLoop = async ({ db }: { db: DrizzleCli }) => {
QueueUrl: QUEUE_URL,
MaxNumberOfMessages: 10, // Receive up to 10 messages at once
WaitTimeSeconds: 20, // Long polling
VisibilityTimeout: 43200, // 12 hours (max) - prevents duplicate processing of long-running jobs
VisibilityTimeout: 30, // 12 hours (max) - prevents duplicate processing of long-running jobs
// For FIFO queues, add ReceiveRequestAttemptId for deduplication
...(isFifoQueue && {
ReceiveRequestAttemptId: generateId("receive"),