diff --git a/apps/web/app/[locale]/docs/getting-started/page.tsx b/apps/web/app/[locale]/docs/getting-started/page.tsx index 09d711e..b606de0 100644 --- a/apps/web/app/[locale]/docs/getting-started/page.tsx +++ b/apps/web/app/[locale]/docs/getting-started/page.tsx @@ -14,10 +14,11 @@ async function getSkillCount(): Promise { const db = createDb(); const result = await db .select({ count: sql`count(*)::int` }) - .from(skills); - return formatPromptSkillCount(result[0]?.count ?? 170000); + .from(skills) + .where(sql`${skills.isDuplicate} = false AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`); + return formatPromptSkillCount(result[0]?.count ?? 16000); } catch { - return '170,000+'; + return '16,000+'; } } diff --git a/apps/web/app/[locale]/page.tsx b/apps/web/app/[locale]/page.tsx index beca19d..83ca8ea 100644 --- a/apps/web/app/[locale]/page.tsx +++ b/apps/web/app/[locale]/page.tsx @@ -44,11 +44,19 @@ async function getStats() { .where(browseReady); const totalContributors = contributorsResult[0]?.count ?? 0; + // Get total indexed skills (all, before curation) for curation note + const totalIndexedResult = await db + .select({ count: sql`count(*)::int` }) + .from(skills) + .where(sql`${skills.isBlocked} = false`); + const totalIndexed = totalIndexedResult[0]?.count ?? 0; + return { totalSkills, totalDownloads, totalCategories, totalContributors, + totalIndexed, platforms: 5, }; } catch (error) { @@ -186,6 +194,9 @@ export default async function HomePage({ ))} +

+ {t('stats.curationNote', { totalIndexed: formatCompactNumber(statsData?.totalIndexed ?? 0, locale) })} +

diff --git a/apps/web/components/BetaBanner.tsx b/apps/web/components/BetaBanner.tsx index 98503b7..2cce997 100644 --- a/apps/web/components/BetaBanner.tsx +++ b/apps/web/components/BetaBanner.tsx @@ -35,7 +35,7 @@ export function BetaBanner() { if (!isVisible) return null; - const count = skillCount || (locale === 'fa' ? '۱۷۲k' : '172k'); + const count = skillCount || (locale === 'fa' ? '۱۶k' : '16k'); return (
diff --git a/apps/web/lib/email-templates.ts b/apps/web/lib/email-templates.ts index 714bd36..9c3a1d1 100644 --- a/apps/web/lib/email-templates.ts +++ b/apps/web/lib/email-templates.ts @@ -220,12 +220,12 @@ export function buildWelcomeEmail(locale: Locale, username: string, email?: stri const dir = getDir(locale); const align = getAlign(locale); - // Format skill count: dynamic if provided, fallback to "172,000+" + // Format skill count: dynamic if provided, fallback to "16,000+" const skillCountStr = totalSkillCount ? (locale === 'fa' ? totalSkillCount.toLocaleString('fa-IR') : totalSkillCount.toLocaleString('en-US')) - : (locale === 'fa' ? '۱۷۲,۰۰۰' : '172,000'); + : (locale === 'fa' ? '۱۶,۰۰۰' : '16,000'); const body = `

diff --git a/apps/web/messages/en.json b/apps/web/messages/en.json index 0a6d27d..e05cbc5 100644 --- a/apps/web/messages/en.json +++ b/apps/web/messages/en.json @@ -47,10 +47,11 @@ "ctaSecondary": "Getting Started" }, "stats": { - "skills": "Skills", + "skills": "Curated Skills", "downloads": "Downloads", "contributors": "Contributors", - "categories": "Categories" + "categories": "Categories", + "curationNote": "Every skill is deduplicated and quality-filtered from {totalIndexed}+ indexed repositories" }, "featured": { "title": "Featured Skills", diff --git a/apps/web/messages/fa.json b/apps/web/messages/fa.json index f371085..3802cf5 100644 --- a/apps/web/messages/fa.json +++ b/apps/web/messages/fa.json @@ -47,10 +47,11 @@ "ctaSecondary": "راهنمای شروع" }, "stats": { - "skills": "مهارت", + "skills": "مهارت برگزیده", "downloads": "دانلود", "contributors": "مشارکت‌کننده", - "categories": "دسته‌بندی" + "categories": "دسته‌بندی", + "curationNote": "هر مهارت از میان {totalIndexed}+ مخزن ایندکس‌شده، حذف تکراری و فیلتر کیفیت شده است" }, "featured": { "title": "مهارت‌های ویژه", diff --git a/packages/db/src/queries.ts b/packages/db/src/queries.ts index 8d495c2..ebaf745 100644 --- a/packages/db/src/queries.ts +++ b/packages/db/src/queries.ts @@ -1128,6 +1128,86 @@ export const categoryQueries = { }, }; +/** + * Post-crawl curation: lightweight classification and dedup + * Runs automatically after each crawl to classify new skills + */ +export async function runPostCrawlCuration(db: DB): Promise<{ classified: number; duplicates: number; categoryCounts: number }> { + console.log('[curation] Running post-crawl curation...'); + const results = { classified: 0, duplicates: 0, categoryCounts: 0 }; + + // Step 1: Update repo_skill_count for skills that don't have it + await db.execute(sql` + UPDATE skills s SET repo_skill_count = sub.cnt + FROM ( + SELECT github_owner, github_repo, COUNT(*)::int AS cnt + FROM skills WHERE is_blocked = false + GROUP BY github_owner, github_repo + ) sub + WHERE s.github_owner = sub.github_owner + AND s.github_repo = sub.github_repo + AND (s.repo_skill_count IS NULL OR s.repo_skill_count != sub.cnt) + `); + + // Step 2: Mark aggregators (repos with 50+ skills) + await db.execute(sql` + UPDATE skills SET skill_type = 'aggregator' + WHERE repo_skill_count >= 50 + AND (skill_type IS NULL OR skill_type != 'aggregator') + AND is_blocked = false + `); + + // Step 3: Classify remaining unclassified skills + const classResult = await db.execute(sql` + UPDATE skills SET skill_type = CASE + WHEN repo_skill_count = 1 THEN 'standalone' + WHEN repo_skill_count BETWEEN 2 AND 49 THEN 'collection' + ELSE 'standalone' + END + WHERE skill_type IS NULL + AND is_blocked = false + AND repo_skill_count IS NOT NULL + `); + results.classified = (classResult as unknown as { rowCount: number }).rowCount ?? 0; + + // Step 4: Mark duplicates by content_hash (keep canonical = most stars) + const dupResult = await db.execute(sql` + UPDATE skills s SET is_duplicate = true + WHERE s.is_blocked = false + AND s.is_duplicate = false + AND s.content_hash IS NOT NULL + AND EXISTS ( + SELECT 1 FROM skills s2 + WHERE s2.content_hash = s.content_hash + AND s2.is_blocked = false + AND s2.id != s.id + AND (s2.github_stars > s.github_stars + OR (s2.github_stars = s.github_stars AND s2.indexed_at < s.indexed_at)) + ) + `); + results.duplicates = (dupResult as unknown as { rowCount: number }).rowCount ?? 0; + + // Step 5: Recalculate category counts (browse-ready only) + const catResult = await db.execute(sql` + UPDATE categories c SET skill_count = sub.cnt + FROM ( + SELECT sc.category_id, COUNT(*)::int AS cnt + FROM skill_categories sc + JOIN skills s ON sc.skill_id = s.id + WHERE s.is_blocked = false + AND s.is_duplicate = false + AND (s.skill_type IS NULL OR s.skill_type != 'aggregator') + GROUP BY sc.category_id + ) sub + WHERE c.id = sub.category_id + AND c.skill_count IS DISTINCT FROM sub.cnt + `); + results.categoryCounts = (catResult as unknown as { rowCount: number }).rowCount ?? 0; + + console.log(`[curation] Done: ${results.classified} classified, ${results.duplicates} new duplicates, ${results.categoryCounts} category counts updated`); + return results; +} + /** * User queries */ diff --git a/services/indexer/src/worker.ts b/services/indexer/src/worker.ts index a978946..1fd91fa 100644 --- a/services/indexer/src/worker.ts +++ b/services/indexer/src/worker.ts @@ -1,7 +1,7 @@ import type { Job } from 'bullmq'; import { Worker } from 'bullmq'; import pLimit from 'p-limit'; -import { createDb, type Database, discoveredRepoQueries, awesomeListQueries, addRequestQueries, skillQueries, sql } from '@skillhub/db'; +import { createDb, type Database, discoveredRepoQueries, awesomeListQueries, addRequestQueries, skillQueries, runPostCrawlCuration, sql } from '@skillhub/db'; import { GitHubCrawler, createCrawler } from './crawler.js'; import type { IndexJobData, IndexJobResult } from './queue.js'; import { setupRecurringJobs } from './queue.js'; @@ -167,6 +167,15 @@ async function processFullCrawl( ); await Promise.all(indexPromises); + + // Post-crawl curation: classify new skills, mark duplicates, update category counts + try { + const db = getDb(); + await runPostCrawlCuration(db); + } catch (error) { + console.error('[curation] Post-crawl curation failed (non-fatal):', error); + } + await job.updateProgress(100); return { @@ -224,6 +233,15 @@ async function processIncrementalCrawl( ); await Promise.all(indexPromises); + + // Post-crawl curation: classify new skills, mark duplicates, update category counts + try { + const db = getDb(); + await runPostCrawlCuration(db); + } catch (error) { + console.error('[curation] Post-crawl curation failed (non-fatal):', error); + } + await job.updateProgress(100); return {