fix(indexer): update skillPath/branch on upsert to fix stale paths
The upsert query was missing skillPath and branch in onConflictDoUpdate, so when a repo was restructured (e.g., skills moved to category subdirectories), the DB retained the old path forever. Also updated skill-indexer to not skip re-indexing when only the path changed. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -15,6 +15,9 @@ import {
|
|||||||
addRequests,
|
addRequests,
|
||||||
emailSubscriptions,
|
emailSubscriptions,
|
||||||
} from './schema.js';
|
} from './schema.js';
|
||||||
|
// #region private-only
|
||||||
|
import { outreachEmails } from './schema.js';
|
||||||
|
// #endregion private-only
|
||||||
|
|
||||||
import type * as schema from './schema.js';
|
import type * as schema from './schema.js';
|
||||||
|
|
||||||
@@ -569,6 +572,8 @@ export const skillQueries = {
|
|||||||
set: {
|
set: {
|
||||||
name: skill.name,
|
name: skill.name,
|
||||||
description: skill.description,
|
description: skill.description,
|
||||||
|
skillPath: skill.skillPath,
|
||||||
|
branch: skill.branch,
|
||||||
version: skill.version,
|
version: skill.version,
|
||||||
license: skill.license,
|
license: skill.license,
|
||||||
author: skill.author,
|
author: skill.author,
|
||||||
@@ -2022,3 +2027,295 @@ export const emailSubscriptionQueries = {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// #region private-only
|
||||||
|
/**
|
||||||
|
* Outreach email queries
|
||||||
|
*/
|
||||||
|
export const outreachQueries = {
|
||||||
|
/**
|
||||||
|
* Upsert an outreach record for a GitHub owner
|
||||||
|
*/
|
||||||
|
upsert: async (
|
||||||
|
db: DB,
|
||||||
|
data: {
|
||||||
|
githubUsername: string;
|
||||||
|
email?: string | null;
|
||||||
|
emailSource?: string | null;
|
||||||
|
repoCount?: number;
|
||||||
|
skillCount?: number;
|
||||||
|
totalStars?: number;
|
||||||
|
status?: string;
|
||||||
|
templateData?: typeof outreachEmails.$inferInsert['templateData'];
|
||||||
|
}
|
||||||
|
) => {
|
||||||
|
const result = await db
|
||||||
|
.insert(outreachEmails)
|
||||||
|
.values({
|
||||||
|
id: crypto.randomUUID(),
|
||||||
|
githubUsername: data.githubUsername,
|
||||||
|
email: data.email ?? undefined,
|
||||||
|
emailSource: data.emailSource ?? undefined,
|
||||||
|
repoCount: data.repoCount ?? 0,
|
||||||
|
skillCount: data.skillCount ?? 0,
|
||||||
|
totalStars: data.totalStars ?? 0,
|
||||||
|
status: data.status ?? (data.email ? 'pending' : 'no_email'),
|
||||||
|
templateData: data.templateData ?? undefined,
|
||||||
|
})
|
||||||
|
.onConflictDoUpdate({
|
||||||
|
target: outreachEmails.githubUsername,
|
||||||
|
set: {
|
||||||
|
...(data.email !== undefined ? { email: data.email } : {}),
|
||||||
|
...(data.emailSource !== undefined ? { emailSource: data.emailSource } : {}),
|
||||||
|
...(data.repoCount !== undefined ? { repoCount: data.repoCount } : {}),
|
||||||
|
...(data.skillCount !== undefined ? { skillCount: data.skillCount } : {}),
|
||||||
|
...(data.totalStars !== undefined ? { totalStars: data.totalStars } : {}),
|
||||||
|
...(data.status !== undefined ? { status: data.status } : {}),
|
||||||
|
...(data.templateData !== undefined ? { templateData: data.templateData } : {}),
|
||||||
|
updatedAt: new Date(),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.returning();
|
||||||
|
return result[0];
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get pending outreach emails (with email, not yet sent)
|
||||||
|
* Three-tier priority:
|
||||||
|
* Tier 0: owners with 1..priorityMaxStars (sweet spot, most likely to engage)
|
||||||
|
* Tier 1: owners with >priorityMaxStars (big orgs, less likely to engage)
|
||||||
|
* Tier 2: owners with 0 stars (least valuable, sent last)
|
||||||
|
* Within each tier, sorted by stars descending
|
||||||
|
*/
|
||||||
|
getPending: async (db: DB, limit: number = 70, priorityMaxStars: number = 10000) => {
|
||||||
|
return db
|
||||||
|
.select()
|
||||||
|
.from(outreachEmails)
|
||||||
|
.where(
|
||||||
|
and(
|
||||||
|
eq(outreachEmails.status, 'pending'),
|
||||||
|
sql`${outreachEmails.email} IS NOT NULL`
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.orderBy(
|
||||||
|
sql`CASE
|
||||||
|
WHEN ${outreachEmails.totalStars} BETWEEN 1 AND ${priorityMaxStars} THEN 0
|
||||||
|
WHEN ${outreachEmails.totalStars} > ${priorityMaxStars} THEN 1
|
||||||
|
ELSE 2
|
||||||
|
END`,
|
||||||
|
desc(outreachEmails.totalStars)
|
||||||
|
)
|
||||||
|
.limit(limit);
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mark an outreach email as sent
|
||||||
|
*/
|
||||||
|
markSent: async (db: DB, githubUsername: string, resendEmailId?: string) => {
|
||||||
|
return db
|
||||||
|
.update(outreachEmails)
|
||||||
|
.set({
|
||||||
|
status: 'sent',
|
||||||
|
sentAt: new Date(),
|
||||||
|
resendEmailId: resendEmailId ?? undefined,
|
||||||
|
})
|
||||||
|
.where(eq(outreachEmails.githubUsername, githubUsername));
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reset falsely sent records back to pending (where resend_email_id is null)
|
||||||
|
* Used to fix records that were marked sent but never actually sent
|
||||||
|
*/
|
||||||
|
resetFalselySent: async (db: DB) => {
|
||||||
|
const result = await db
|
||||||
|
.update(outreachEmails)
|
||||||
|
.set({
|
||||||
|
status: 'pending',
|
||||||
|
sentAt: null,
|
||||||
|
})
|
||||||
|
.where(
|
||||||
|
and(
|
||||||
|
eq(outreachEmails.status, 'sent'),
|
||||||
|
sql`${outreachEmails.resendEmailId} IS NULL`
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.returning({ githubUsername: outreachEmails.githubUsername });
|
||||||
|
return result.length;
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reset failed records back to pending (e.g. after rate limiting)
|
||||||
|
* Optionally filter by error message pattern
|
||||||
|
*/
|
||||||
|
resetFailed: async (db: DB, errorPattern?: string) => {
|
||||||
|
const conditions = [eq(outreachEmails.status, 'failed')];
|
||||||
|
if (errorPattern) {
|
||||||
|
conditions.push(like(outreachEmails.errorMessage, `%${errorPattern}%`));
|
||||||
|
}
|
||||||
|
const result = await db
|
||||||
|
.update(outreachEmails)
|
||||||
|
.set({
|
||||||
|
status: 'pending',
|
||||||
|
errorMessage: null,
|
||||||
|
})
|
||||||
|
.where(and(...conditions))
|
||||||
|
.returning({ githubUsername: outreachEmails.githubUsername });
|
||||||
|
return result.length;
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mark an outreach email as failed
|
||||||
|
*/
|
||||||
|
markFailed: async (db: DB, githubUsername: string, error: string) => {
|
||||||
|
return db
|
||||||
|
.update(outreachEmails)
|
||||||
|
.set({
|
||||||
|
status: 'failed',
|
||||||
|
errorMessage: error,
|
||||||
|
})
|
||||||
|
.where(eq(outreachEmails.githubUsername, githubUsername));
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mark an outreach email as unsubscribed
|
||||||
|
*/
|
||||||
|
markUnsubscribed: async (db: DB, githubUsername: string) => {
|
||||||
|
return db
|
||||||
|
.update(outreachEmails)
|
||||||
|
.set({ status: 'unsubscribed' })
|
||||||
|
.where(eq(outreachEmails.githubUsername, githubUsername));
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get outreach stats by status
|
||||||
|
*/
|
||||||
|
getStats: async (db: DB) => {
|
||||||
|
return db
|
||||||
|
.select({
|
||||||
|
status: outreachEmails.status,
|
||||||
|
count: sql<number>`count(*)::int`,
|
||||||
|
})
|
||||||
|
.from(outreachEmails)
|
||||||
|
.groupBy(outreachEmails.status);
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get star distribution for pending outreach emails (with email)
|
||||||
|
* Useful for understanding the priority tiers
|
||||||
|
*/
|
||||||
|
getStarDistribution: async (db: DB) => {
|
||||||
|
return db.execute(sql`
|
||||||
|
SELECT
|
||||||
|
CASE
|
||||||
|
WHEN total_stars = 0 THEN '0'
|
||||||
|
WHEN total_stars BETWEEN 1 AND 100 THEN '1-100'
|
||||||
|
WHEN total_stars BETWEEN 101 AND 500 THEN '101-500'
|
||||||
|
WHEN total_stars BETWEEN 501 AND 1000 THEN '501-1K'
|
||||||
|
WHEN total_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
|
||||||
|
WHEN total_stars BETWEEN 5001 AND 10000 THEN '5K-10K'
|
||||||
|
WHEN total_stars BETWEEN 10001 AND 50000 THEN '10K-50K'
|
||||||
|
ELSE '50K+'
|
||||||
|
END as star_range,
|
||||||
|
COUNT(*) as total,
|
||||||
|
COUNT(*) FILTER (WHERE status = 'pending' AND email IS NOT NULL) as sendable,
|
||||||
|
COUNT(*) FILTER (WHERE status = 'sent') as sent,
|
||||||
|
COUNT(*) FILTER (WHERE status = 'no_email') as no_email
|
||||||
|
FROM outreach_emails
|
||||||
|
GROUP BY
|
||||||
|
CASE
|
||||||
|
WHEN total_stars = 0 THEN '0'
|
||||||
|
WHEN total_stars BETWEEN 1 AND 100 THEN '1-100'
|
||||||
|
WHEN total_stars BETWEEN 101 AND 500 THEN '101-500'
|
||||||
|
WHEN total_stars BETWEEN 501 AND 1000 THEN '501-1K'
|
||||||
|
WHEN total_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
|
||||||
|
WHEN total_stars BETWEEN 5001 AND 10000 THEN '5K-10K'
|
||||||
|
WHEN total_stars BETWEEN 10001 AND 50000 THEN '10K-50K'
|
||||||
|
ELSE '50K+'
|
||||||
|
END
|
||||||
|
ORDER BY MIN(total_stars)
|
||||||
|
`);
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Populate outreach_emails from skills table (aggregated by owner)
|
||||||
|
* Only inserts new owners that don't already exist in outreach_emails
|
||||||
|
*/
|
||||||
|
populateFromSkills: async (_db: DB) => {
|
||||||
|
const rawClient = getRawClientLocal();
|
||||||
|
const result = await rawClient`
|
||||||
|
INSERT INTO outreach_emails (id, github_username, repo_count, skill_count, total_stars, status, template_data, created_at, updated_at)
|
||||||
|
SELECT
|
||||||
|
gen_random_uuid()::TEXT,
|
||||||
|
s.github_owner,
|
||||||
|
COUNT(DISTINCT s.github_repo)::INTEGER,
|
||||||
|
COUNT(*)::INTEGER,
|
||||||
|
COALESCE(MAX(s.github_stars), 0),
|
||||||
|
'pending',
|
||||||
|
jsonb_build_object(
|
||||||
|
'repos', (
|
||||||
|
SELECT jsonb_agg(repo_data ORDER BY repo_data->>'stars' DESC)
|
||||||
|
FROM (
|
||||||
|
SELECT jsonb_build_object(
|
||||||
|
'name', sub.github_repo,
|
||||||
|
'skillCount', sub.cnt,
|
||||||
|
'stars', sub.max_stars,
|
||||||
|
'topSkills', sub.top_skills
|
||||||
|
) as repo_data
|
||||||
|
FROM (
|
||||||
|
SELECT
|
||||||
|
s2.github_repo,
|
||||||
|
COUNT(*)::INTEGER as cnt,
|
||||||
|
MAX(s2.github_stars) as max_stars,
|
||||||
|
jsonb_agg(
|
||||||
|
jsonb_build_object('id', s2.id, 'name', s2.name, 'description', LEFT(s2.description, 100))
|
||||||
|
ORDER BY s2.github_stars DESC
|
||||||
|
) FILTER (WHERE s2.rn <= 3) as top_skills
|
||||||
|
FROM (
|
||||||
|
SELECT *, ROW_NUMBER() OVER (PARTITION BY github_owner, github_repo ORDER BY github_stars DESC) as rn
|
||||||
|
FROM skills
|
||||||
|
WHERE github_owner = s.github_owner AND is_blocked = false
|
||||||
|
) s2
|
||||||
|
GROUP BY s2.github_repo
|
||||||
|
) sub
|
||||||
|
) repo_sub
|
||||||
|
)
|
||||||
|
),
|
||||||
|
NOW(),
|
||||||
|
NOW()
|
||||||
|
FROM skills s
|
||||||
|
WHERE s.is_blocked = false
|
||||||
|
GROUP BY s.github_owner
|
||||||
|
ON CONFLICT (github_username) DO UPDATE SET
|
||||||
|
repo_count = EXCLUDED.repo_count,
|
||||||
|
skill_count = EXCLUDED.skill_count,
|
||||||
|
total_stars = EXCLUDED.total_stars,
|
||||||
|
template_data = EXCLUDED.template_data,
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE outreach_emails.status NOT IN ('sent', 'unsubscribed')
|
||||||
|
`;
|
||||||
|
return result.count;
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get by github username
|
||||||
|
*/
|
||||||
|
getByUsername: async (db: DB, githubUsername: string) => {
|
||||||
|
const result = await db
|
||||||
|
.select()
|
||||||
|
.from(outreachEmails)
|
||||||
|
.where(eq(outreachEmails.githubUsername, githubUsername))
|
||||||
|
.limit(1);
|
||||||
|
return result[0] ?? null;
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get total skill count (for dynamic numbers in emails)
|
||||||
|
*/
|
||||||
|
getTotalSkillCount: async (db: DB): Promise<number> => {
|
||||||
|
const result = await db
|
||||||
|
.select({ count: sql<number>`count(*)::int` })
|
||||||
|
.from(skills)
|
||||||
|
.where(eq(skills.isBlocked, false));
|
||||||
|
return result[0]?.count ?? 0;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
// #endregion private-only
|
||||||
|
|||||||
@@ -58,10 +58,18 @@ export async function indexSkill(
|
|||||||
// Check if we need to update (unless force)
|
// Check if we need to update (unless force)
|
||||||
if (!force) {
|
if (!force) {
|
||||||
if (existing && existing.contentHash === analysis.meta.contentHash) {
|
if (existing && existing.contentHash === analysis.meta.contentHash) {
|
||||||
|
// Content unchanged, but check if path/branch changed
|
||||||
|
// (e.g., repo was restructured, file moved to a different directory)
|
||||||
|
const actualBranch = source.branch || content.repoMeta.defaultBranch;
|
||||||
|
if (existing.skillPath !== source.path || existing.branch !== actualBranch) {
|
||||||
|
console.log(`Skill ${skillId} path changed: ${existing.skillPath} → ${source.path}`);
|
||||||
|
// Don't skip - fall through to upsert which now updates skillPath/branch
|
||||||
|
} else {
|
||||||
console.log(`Skill ${skillId} unchanged, skipping`);
|
console.log(`Skill ${skillId} unchanged, skipping`);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Upsert to database
|
// Upsert to database
|
||||||
await skillQueries.upsert(database, {
|
await skillQueries.upsert(database, {
|
||||||
|
|||||||
Reference in New Issue
Block a user