feat(curation): data quality pipeline — Phases 1-3

Add comprehensive data curation system to clean up the 197K skill
dataset and show only quality browse-ready skills to users.

Phase 1 — Database exploration:
- Explore scripts (explore.ts, explore.mjs, explore.sql) for analysis
- Discovered: 69% duplicates, 77% aggregator/fork noise

Phase 2 — Data cleanup and classification:
- Schema: 6 new curation columns + 4 indexes
- curate.mjs: 8-step pipeline (classify, dedup, fork detection, etc.)
- Result: 197K → 60K unique → 16K browse-ready skills
- Bug fix: securityStatus was computed but never stored during crawl

Phase 3 — UI browse-ready filters:
- browseReadyFilter applied to 17+ query functions
- Homepage stats show accurate browse-ready counts
- Stats API filtered (previously had no WHERE clause)
- Category counts recalculated (e.g. 45K → 3.1K)
- Featured skills exclude duplicates and aggregators

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

apps/web/app/[locale]/page.tsx

@@ -16,27 +16,32 @@ async function getStats() {
   try {
     const db = createDb();
 
-    // Get total skills count (SKILL.md only - real reusable skills)
+    // Browse-ready filter: exclude duplicates and aggregators
+    const browseReady = sql`${skills.isDuplicate} = false AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`;
+
+    // Get total skills count (browse-ready, SKILL.md only)
     const skillsResult = await db
       .select({ count: sql<number>`count(*)::int` })
       .from(skills)
-      .where(sql`${skills.sourceFormat} = 'skill.md' AND ${skills.isBlocked} = false`);
+      .where(sql`${skills.sourceFormat} = 'skill.md' AND ${skills.isBlocked} = false AND ${browseReady}`);
     const totalSkills = skillsResult[0]?.count ?? 0;
 
-    // Get total downloads
+    // Get total downloads (browse-ready only)
     const downloadsResult = await db
       .select({ sum: sql<number>`coalesce(sum(${skills.downloadCount}), 0)::int` })
-      .from(skills);
+      .from(skills)
+      .where(browseReady);
     const totalDownloads = downloadsResult[0]?.sum ?? 0;
 
     // Get total categories
     const categories = await categoryQueries.getAll(db);
     const totalCategories = categories.length;
 
-    // Get unique contributors (github owners)
+    // Get unique contributors (browse-ready skills only)
     const contributorsResult = await db
       .select({ count: sql<number>`count(distinct ${skills.githubOwner})::int` })
-      .from(skills);
+      .from(skills)
+      .where(browseReady);
     const totalContributors = contributorsResult[0]?.count ?? 0;
 
     return {

apps/web/app/api/stats/route.test.ts

@@ -18,17 +18,26 @@ vi.mock('@/lib/cache', () => ({
 
 // Mock the db module - must be before imports that use it
 vi.mock('@skillhub/db', () => {
+  const mockStatsRow = [{ totalSkills: 100, totalDownloads: 5000, totalContributors: 50 }];
+  const mockCategoryRow = [{ count: 8 }];
   return {
     createDb: vi.fn(() => ({
       select: vi.fn().mockReturnValue({
-        from: vi.fn().mockResolvedValue([
-          { totalSkills: 100, totalDownloads: 5000, totalContributors: 50 },
-        ]),
+        from: vi.fn((table: unknown) => {
+          // categories table query returns count directly (no .where())
+          if (table === 'categories-table') {
+            return Promise.resolve(mockCategoryRow);
+          }
+          // skills table query has .where() chained
+          return {
+            where: vi.fn().mockResolvedValue(mockStatsRow),
+          };
+        }),
       }),
     })),
-    skills: { downloadCount: 'download_count', githubOwner: 'github_owner' },
-    categories: {},
-    sql: vi.fn(() => 'mock-sql'),
+    skills: { downloadCount: 'download_count', githubOwner: 'github_owner', isDuplicate: 'is_duplicate', skillType: 'skill_type' },
+    categories: 'categories-table',
+    sql: vi.fn((..._args: unknown[]) => 'mock-sql'),
   };
 });
 

apps/web/app/api/stats/route.ts

@@ -34,14 +34,18 @@ export async function GET(request: NextRequest) {
       });
     }
 
-    // Consolidate all skill stats into a single query for better performance
+    // Browse-ready filter: exclude duplicates and aggregators
+    const browseReady = sql`${skills.isDuplicate} = false AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`;
+
+    // Consolidate all skill stats into a single query (browse-ready only)
     const statsResult = await db
       .select({
         totalSkills: sql<number>`count(*)::int`,
         totalDownloads: sql<number>`coalesce(sum(${skills.downloadCount}), 0)::int`,
         totalContributors: sql<number>`count(distinct ${skills.githubOwner})::int`,
       })
-      .from(skills);
+      .from(skills)
+      .where(browseReady);
 
     // Get category count in separate query (different table)
     const categoryResult = await db