feat(curation): auto-curation after crawl + quality messaging

Automate curation:
- Add runPostCrawlCuration() to DB queries — runs after every
  full and incremental crawl (classify, dedup, category counts)
- No more need for manual curate.mjs runs after each crawl

Quality messaging:
- Homepage stats label: "Skills" → "Curated Skills" (en/fa)
- Add curation note: "deduplicated and quality-filtered from X+ repos"
- Update fallback counts from 172K to 16K across all touchpoints
  (BetaBanner, getting-started prompts, email templates)
- getting-started skill count query now uses browse-ready filter

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

apps/web/app/[locale]/docs/getting-started/page.tsx

@@ -14,10 +14,11 @@ async function getSkillCount(): Promise<string> {
     const db = createDb();
     const result = await db
       .select({ count: sql<number>`count(*)::int` })
-      .from(skills);
+      .from(skills)
-    return formatPromptSkillCount(result[0]?.count ?? 170000);
+      .where(sql`${skills.isDuplicate} = false AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`);
+    return formatPromptSkillCount(result[0]?.count ?? 16000);
   } catch {
-    return '170,000+';
+    return '16,000+';
   }
 }
 

apps/web/app/[locale]/page.tsx

@@ -44,11 +44,19 @@ async function getStats() {
       .where(browseReady);
     const totalContributors = contributorsResult[0]?.count ?? 0;
 
+    // Get total indexed skills (all, before curation) for curation note
+    const totalIndexedResult = await db
+      .select({ count: sql<number>`count(*)::int` })
+      .from(skills)
+      .where(sql`${skills.isBlocked} = false`);
+    const totalIndexed = totalIndexedResult[0]?.count ?? 0;
+
     return {
       totalSkills,
       totalDownloads,
       totalCategories,
       totalContributors,
+      totalIndexed,
       platforms: 5,
     };
   } catch (error) {
@@ -186,6 +194,9 @@ export default async function HomePage({
                 </div>
               ))}
             </div>
+            <p className="text-center text-text-muted text-sm mt-6">
+              {t('stats.curationNote', { totalIndexed: formatCompactNumber(statsData?.totalIndexed ?? 0, locale) })}
+            </p>
           </div>
         </section>
 

apps/web/components/BetaBanner.tsx

@@ -35,7 +35,7 @@ export function BetaBanner() {
 
   if (!isVisible) return null;
 
-  const count = skillCount || (locale === 'fa' ? '۱۷۲k' : '172k');
+  const count = skillCount || (locale === 'fa' ? '۱۶k' : '16k');
 
   return (
     <div className="relative bg-gradient-to-r from-primary-600 to-primary-500 text-white">

apps/web/lib/email-templates.ts

@@ -220,12 +220,12 @@ export function buildWelcomeEmail(locale: Locale, username: string, email?: stri
   const dir = getDir(locale);
   const align = getAlign(locale);
 
-  // Format skill count: dynamic if provided, fallback to "172,000+"
+  // Format skill count: dynamic if provided, fallback to "16,000+"
   const skillCountStr = totalSkillCount
     ? (locale === 'fa'
       ? totalSkillCount.toLocaleString('fa-IR')
       : totalSkillCount.toLocaleString('en-US'))
-    : (locale === 'fa' ? '۱۷۲,۰۰۰' : '172,000');
+    : (locale === 'fa' ? '۱۶,۰۰۰' : '16,000');
 
   const body = `
     <h1 style="margin: 0 0 16px 0; color: ${COLORS.text}; font-size: 24px; text-align: ${align};">

apps/web/messages/en.json

@@ -47,10 +47,11 @@
       "ctaSecondary": "Getting Started"
     },
     "stats": {
-      "skills": "Skills",
+      "skills": "Curated Skills",
       "downloads": "Downloads",
       "contributors": "Contributors",
-      "categories": "Categories"
+      "categories": "Categories",
+      "curationNote": "Every skill is deduplicated and quality-filtered from {totalIndexed}+ indexed repositories"
     },
     "featured": {
       "title": "Featured Skills",

apps/web/messages/fa.json

@@ -47,10 +47,11 @@
       "ctaSecondary": "راهنمای شروع"
     },
     "stats": {
-      "skills": "مهارت",
+      "skills": "مهارت برگزیده",
       "downloads": "دانلود",
       "contributors": "مشارکت‌کننده",
-      "categories": "دسته‌بندی"
+      "categories": "دسته‌بندی",
+      "curationNote": "هر مهارت از میان {totalIndexed}+ مخزن ایندکس‌شده، حذف تکراری و فیلتر کیفیت شده است"
     },
     "featured": {
       "title": "مهارت‌های ویژه",

packages/db/src/queries.ts

@@ -1128,6 +1128,86 @@ export const categoryQueries = {
   },
 };
 
+/**
+ * Post-crawl curation: lightweight classification and dedup
+ * Runs automatically after each crawl to classify new skills
+ */
+export async function runPostCrawlCuration(db: DB): Promise<{ classified: number; duplicates: number; categoryCounts: number }> {
+  console.log('[curation] Running post-crawl curation...');
+  const results = { classified: 0, duplicates: 0, categoryCounts: 0 };
+
+  // Step 1: Update repo_skill_count for skills that don't have it
+  await db.execute(sql`
+    UPDATE skills s SET repo_skill_count = sub.cnt
+    FROM (
+      SELECT github_owner, github_repo, COUNT(*)::int AS cnt
+      FROM skills WHERE is_blocked = false
+      GROUP BY github_owner, github_repo
+    ) sub
+    WHERE s.github_owner = sub.github_owner
+      AND s.github_repo = sub.github_repo
+      AND (s.repo_skill_count IS NULL OR s.repo_skill_count != sub.cnt)
+  `);
+
+  // Step 2: Mark aggregators (repos with 50+ skills)
+  await db.execute(sql`
+    UPDATE skills SET skill_type = 'aggregator'
+    WHERE repo_skill_count >= 50
+      AND (skill_type IS NULL OR skill_type != 'aggregator')
+      AND is_blocked = false
+  `);
+
+  // Step 3: Classify remaining unclassified skills
+  const classResult = await db.execute(sql`
+    UPDATE skills SET skill_type = CASE
+      WHEN repo_skill_count = 1 THEN 'standalone'
+      WHEN repo_skill_count BETWEEN 2 AND 49 THEN 'collection'
+      ELSE 'standalone'
+    END
+    WHERE skill_type IS NULL
+      AND is_blocked = false
+      AND repo_skill_count IS NOT NULL
+  `);
+  results.classified = (classResult as unknown as { rowCount: number }).rowCount ?? 0;
+
+  // Step 4: Mark duplicates by content_hash (keep canonical = most stars)
+  const dupResult = await db.execute(sql`
+    UPDATE skills s SET is_duplicate = true
+    WHERE s.is_blocked = false
+      AND s.is_duplicate = false
+      AND s.content_hash IS NOT NULL
+      AND EXISTS (
+        SELECT 1 FROM skills s2
+        WHERE s2.content_hash = s.content_hash
+          AND s2.is_blocked = false
+          AND s2.id != s.id
+          AND (s2.github_stars > s.github_stars
+               OR (s2.github_stars = s.github_stars AND s2.indexed_at < s.indexed_at))
+      )
+  `);
+  results.duplicates = (dupResult as unknown as { rowCount: number }).rowCount ?? 0;
+
+  // Step 5: Recalculate category counts (browse-ready only)
+  const catResult = await db.execute(sql`
+    UPDATE categories c SET skill_count = sub.cnt
+    FROM (
+      SELECT sc.category_id, COUNT(*)::int AS cnt
+      FROM skill_categories sc
+      JOIN skills s ON sc.skill_id = s.id
+      WHERE s.is_blocked = false
+        AND s.is_duplicate = false
+        AND (s.skill_type IS NULL OR s.skill_type != 'aggregator')
+      GROUP BY sc.category_id
+    ) sub
+    WHERE c.id = sub.category_id
+      AND c.skill_count IS DISTINCT FROM sub.cnt
+  `);
+  results.categoryCounts = (catResult as unknown as { rowCount: number }).rowCount ?? 0;
+
+  console.log(`[curation] Done: ${results.classified} classified, ${results.duplicates} new duplicates, ${results.categoryCounts} category counts updated`);
+  return results;
+}
+
 /**
  * User queries
  */

services/indexer/src/worker.ts

@@ -1,7 +1,7 @@
 import type { Job } from 'bullmq';
 import { Worker } from 'bullmq';
 import pLimit from 'p-limit';
-import { createDb, type Database, discoveredRepoQueries, awesomeListQueries, addRequestQueries, skillQueries, sql } from '@skillhub/db';
+import { createDb, type Database, discoveredRepoQueries, awesomeListQueries, addRequestQueries, skillQueries, runPostCrawlCuration, sql } from '@skillhub/db';
 import { GitHubCrawler, createCrawler } from './crawler.js';
 import type { IndexJobData, IndexJobResult } from './queue.js';
 import { setupRecurringJobs } from './queue.js';
@@ -167,6 +167,15 @@ async function processFullCrawl(
   );
 
   await Promise.all(indexPromises);
+
+  // Post-crawl curation: classify new skills, mark duplicates, update category counts
+  try {
+    const db = getDb();
+    await runPostCrawlCuration(db);
+  } catch (error) {
+    console.error('[curation] Post-crawl curation failed (non-fatal):', error);
+  }
+
   await job.updateProgress(100);
 
   return {
@@ -224,6 +233,15 @@ async function processIncrementalCrawl(
   );
 
   await Promise.all(indexPromises);
+
+  // Post-crawl curation: classify new skills, mark duplicates, update category counts
+  try {
+    const db = getDb();
+    await runPostCrawlCuration(db);
+  } catch (error) {
+    console.error('[curation] Post-crawl curation failed (non-fatal):', error);
+  }
+
   await job.updateProgress(100);
 
   return {