From e8bf89d31138b2405b53e18cd1ad4e3caf4f4e0c Mon Sep 17 00:00:00 2001 From: airano Date: Fri, 13 Feb 2026 04:56:40 +0330 Subject: [PATCH] fix(indexer): update skillPath/branch on upsert to fix stale paths The upsert query was missing skillPath and branch in onConflictDoUpdate, so when a repo was restructured (e.g., skills moved to category subdirectories), the DB retained the old path forever. Also updated skill-indexer to not skip re-indexing when only the path changed. Co-Authored-By: Claude Opus 4.6 --- packages/db/src/queries.ts | 297 ++++++++++++++++++++++++++ services/indexer/src/skill-indexer.ts | 12 +- 2 files changed, 307 insertions(+), 2 deletions(-) diff --git a/packages/db/src/queries.ts b/packages/db/src/queries.ts index 7de37f2..702d5f6 100644 --- a/packages/db/src/queries.ts +++ b/packages/db/src/queries.ts @@ -15,6 +15,9 @@ import { addRequests, emailSubscriptions, } from './schema.js'; +// #region private-only +import { outreachEmails } from './schema.js'; +// #endregion private-only import type * as schema from './schema.js'; @@ -569,6 +572,8 @@ export const skillQueries = { set: { name: skill.name, description: skill.description, + skillPath: skill.skillPath, + branch: skill.branch, version: skill.version, license: skill.license, author: skill.author, @@ -2022,3 +2027,295 @@ export const emailSubscriptionQueries = { }, }; +// #region private-only +/** + * Outreach email queries + */ +export const outreachQueries = { + /** + * Upsert an outreach record for a GitHub owner + */ + upsert: async ( + db: DB, + data: { + githubUsername: string; + email?: string | null; + emailSource?: string | null; + repoCount?: number; + skillCount?: number; + totalStars?: number; + status?: string; + templateData?: typeof outreachEmails.$inferInsert['templateData']; + } + ) => { + const result = await db + .insert(outreachEmails) + .values({ + id: crypto.randomUUID(), + githubUsername: data.githubUsername, + email: data.email ?? undefined, + emailSource: data.emailSource ?? undefined, + repoCount: data.repoCount ?? 0, + skillCount: data.skillCount ?? 0, + totalStars: data.totalStars ?? 0, + status: data.status ?? (data.email ? 'pending' : 'no_email'), + templateData: data.templateData ?? undefined, + }) + .onConflictDoUpdate({ + target: outreachEmails.githubUsername, + set: { + ...(data.email !== undefined ? { email: data.email } : {}), + ...(data.emailSource !== undefined ? { emailSource: data.emailSource } : {}), + ...(data.repoCount !== undefined ? { repoCount: data.repoCount } : {}), + ...(data.skillCount !== undefined ? { skillCount: data.skillCount } : {}), + ...(data.totalStars !== undefined ? { totalStars: data.totalStars } : {}), + ...(data.status !== undefined ? { status: data.status } : {}), + ...(data.templateData !== undefined ? { templateData: data.templateData } : {}), + updatedAt: new Date(), + }, + }) + .returning(); + return result[0]; + }, + + /** + * Get pending outreach emails (with email, not yet sent) + * Three-tier priority: + * Tier 0: owners with 1..priorityMaxStars (sweet spot, most likely to engage) + * Tier 1: owners with >priorityMaxStars (big orgs, less likely to engage) + * Tier 2: owners with 0 stars (least valuable, sent last) + * Within each tier, sorted by stars descending + */ + getPending: async (db: DB, limit: number = 70, priorityMaxStars: number = 10000) => { + return db + .select() + .from(outreachEmails) + .where( + and( + eq(outreachEmails.status, 'pending'), + sql`${outreachEmails.email} IS NOT NULL` + ) + ) + .orderBy( + sql`CASE + WHEN ${outreachEmails.totalStars} BETWEEN 1 AND ${priorityMaxStars} THEN 0 + WHEN ${outreachEmails.totalStars} > ${priorityMaxStars} THEN 1 + ELSE 2 + END`, + desc(outreachEmails.totalStars) + ) + .limit(limit); + }, + + /** + * Mark an outreach email as sent + */ + markSent: async (db: DB, githubUsername: string, resendEmailId?: string) => { + return db + .update(outreachEmails) + .set({ + status: 'sent', + sentAt: new Date(), + resendEmailId: resendEmailId ?? undefined, + }) + .where(eq(outreachEmails.githubUsername, githubUsername)); + }, + + /** + * Reset falsely sent records back to pending (where resend_email_id is null) + * Used to fix records that were marked sent but never actually sent + */ + resetFalselySent: async (db: DB) => { + const result = await db + .update(outreachEmails) + .set({ + status: 'pending', + sentAt: null, + }) + .where( + and( + eq(outreachEmails.status, 'sent'), + sql`${outreachEmails.resendEmailId} IS NULL` + ) + ) + .returning({ githubUsername: outreachEmails.githubUsername }); + return result.length; + }, + + /** + * Reset failed records back to pending (e.g. after rate limiting) + * Optionally filter by error message pattern + */ + resetFailed: async (db: DB, errorPattern?: string) => { + const conditions = [eq(outreachEmails.status, 'failed')]; + if (errorPattern) { + conditions.push(like(outreachEmails.errorMessage, `%${errorPattern}%`)); + } + const result = await db + .update(outreachEmails) + .set({ + status: 'pending', + errorMessage: null, + }) + .where(and(...conditions)) + .returning({ githubUsername: outreachEmails.githubUsername }); + return result.length; + }, + + /** + * Mark an outreach email as failed + */ + markFailed: async (db: DB, githubUsername: string, error: string) => { + return db + .update(outreachEmails) + .set({ + status: 'failed', + errorMessage: error, + }) + .where(eq(outreachEmails.githubUsername, githubUsername)); + }, + + /** + * Mark an outreach email as unsubscribed + */ + markUnsubscribed: async (db: DB, githubUsername: string) => { + return db + .update(outreachEmails) + .set({ status: 'unsubscribed' }) + .where(eq(outreachEmails.githubUsername, githubUsername)); + }, + + /** + * Get outreach stats by status + */ + getStats: async (db: DB) => { + return db + .select({ + status: outreachEmails.status, + count: sql`count(*)::int`, + }) + .from(outreachEmails) + .groupBy(outreachEmails.status); + }, + + /** + * Get star distribution for pending outreach emails (with email) + * Useful for understanding the priority tiers + */ + getStarDistribution: async (db: DB) => { + return db.execute(sql` + SELECT + CASE + WHEN total_stars = 0 THEN '0' + WHEN total_stars BETWEEN 1 AND 100 THEN '1-100' + WHEN total_stars BETWEEN 101 AND 500 THEN '101-500' + WHEN total_stars BETWEEN 501 AND 1000 THEN '501-1K' + WHEN total_stars BETWEEN 1001 AND 5000 THEN '1K-5K' + WHEN total_stars BETWEEN 5001 AND 10000 THEN '5K-10K' + WHEN total_stars BETWEEN 10001 AND 50000 THEN '10K-50K' + ELSE '50K+' + END as star_range, + COUNT(*) as total, + COUNT(*) FILTER (WHERE status = 'pending' AND email IS NOT NULL) as sendable, + COUNT(*) FILTER (WHERE status = 'sent') as sent, + COUNT(*) FILTER (WHERE status = 'no_email') as no_email + FROM outreach_emails + GROUP BY + CASE + WHEN total_stars = 0 THEN '0' + WHEN total_stars BETWEEN 1 AND 100 THEN '1-100' + WHEN total_stars BETWEEN 101 AND 500 THEN '101-500' + WHEN total_stars BETWEEN 501 AND 1000 THEN '501-1K' + WHEN total_stars BETWEEN 1001 AND 5000 THEN '1K-5K' + WHEN total_stars BETWEEN 5001 AND 10000 THEN '5K-10K' + WHEN total_stars BETWEEN 10001 AND 50000 THEN '10K-50K' + ELSE '50K+' + END + ORDER BY MIN(total_stars) + `); + }, + + /** + * Populate outreach_emails from skills table (aggregated by owner) + * Only inserts new owners that don't already exist in outreach_emails + */ + populateFromSkills: async (_db: DB) => { + const rawClient = getRawClientLocal(); + const result = await rawClient` + INSERT INTO outreach_emails (id, github_username, repo_count, skill_count, total_stars, status, template_data, created_at, updated_at) + SELECT + gen_random_uuid()::TEXT, + s.github_owner, + COUNT(DISTINCT s.github_repo)::INTEGER, + COUNT(*)::INTEGER, + COALESCE(MAX(s.github_stars), 0), + 'pending', + jsonb_build_object( + 'repos', ( + SELECT jsonb_agg(repo_data ORDER BY repo_data->>'stars' DESC) + FROM ( + SELECT jsonb_build_object( + 'name', sub.github_repo, + 'skillCount', sub.cnt, + 'stars', sub.max_stars, + 'topSkills', sub.top_skills + ) as repo_data + FROM ( + SELECT + s2.github_repo, + COUNT(*)::INTEGER as cnt, + MAX(s2.github_stars) as max_stars, + jsonb_agg( + jsonb_build_object('id', s2.id, 'name', s2.name, 'description', LEFT(s2.description, 100)) + ORDER BY s2.github_stars DESC + ) FILTER (WHERE s2.rn <= 3) as top_skills + FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY github_owner, github_repo ORDER BY github_stars DESC) as rn + FROM skills + WHERE github_owner = s.github_owner AND is_blocked = false + ) s2 + GROUP BY s2.github_repo + ) sub + ) repo_sub + ) + ), + NOW(), + NOW() + FROM skills s + WHERE s.is_blocked = false + GROUP BY s.github_owner + ON CONFLICT (github_username) DO UPDATE SET + repo_count = EXCLUDED.repo_count, + skill_count = EXCLUDED.skill_count, + total_stars = EXCLUDED.total_stars, + template_data = EXCLUDED.template_data, + updated_at = NOW() + WHERE outreach_emails.status NOT IN ('sent', 'unsubscribed') + `; + return result.count; + }, + + /** + * Get by github username + */ + getByUsername: async (db: DB, githubUsername: string) => { + const result = await db + .select() + .from(outreachEmails) + .where(eq(outreachEmails.githubUsername, githubUsername)) + .limit(1); + return result[0] ?? null; + }, + + /** + * Get total skill count (for dynamic numbers in emails) + */ + getTotalSkillCount: async (db: DB): Promise => { + const result = await db + .select({ count: sql`count(*)::int` }) + .from(skills) + .where(eq(skills.isBlocked, false)); + return result[0]?.count ?? 0; + }, +}; +// #endregion private-only diff --git a/services/indexer/src/skill-indexer.ts b/services/indexer/src/skill-indexer.ts index b2b29d1..13e8ae5 100644 --- a/services/indexer/src/skill-indexer.ts +++ b/services/indexer/src/skill-indexer.ts @@ -58,8 +58,16 @@ export async function indexSkill( // Check if we need to update (unless force) if (!force) { if (existing && existing.contentHash === analysis.meta.contentHash) { - console.log(`Skill ${skillId} unchanged, skipping`); - return null; + // Content unchanged, but check if path/branch changed + // (e.g., repo was restructured, file moved to a different directory) + const actualBranch = source.branch || content.repoMeta.defaultBranch; + if (existing.skillPath !== source.path || existing.branch !== actualBranch) { + console.log(`Skill ${skillId} path changed: ${existing.skillPath} → ${source.path}`); + // Don't skip - fall through to upsert which now updates skillPath/branch + } else { + console.log(`Skill ${skillId} unchanged, skipping`); + return null; + } } }