feat(curation): auto-curation after crawl + quality messaging

Automate curation:
- Add runPostCrawlCuration() to DB queries — runs after every
  full and incremental crawl (classify, dedup, category counts)
- No more need for manual curate.mjs runs after each crawl

Quality messaging:
- Homepage stats label: "Skills" → "Curated Skills" (en/fa)
- Add curation note: "deduplicated and quality-filtered from X+ repos"
- Update fallback counts from 172K to 16K across all touchpoints
  (BetaBanner, getting-started prompts, email templates)
- getting-started skill count query now uses browse-ready filter

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
airano
2026-02-19 15:06:18 +03:30
parent caca09fbe7
commit c8216dfcd2
8 changed files with 123 additions and 11 deletions

View File

@@ -1,7 +1,7 @@
import type { Job } from 'bullmq';
import { Worker } from 'bullmq';
import pLimit from 'p-limit';
import { createDb, type Database, discoveredRepoQueries, awesomeListQueries, addRequestQueries, skillQueries, sql } from '@skillhub/db';
import { createDb, type Database, discoveredRepoQueries, awesomeListQueries, addRequestQueries, skillQueries, runPostCrawlCuration, sql } from '@skillhub/db';
import { GitHubCrawler, createCrawler } from './crawler.js';
import type { IndexJobData, IndexJobResult } from './queue.js';
import { setupRecurringJobs } from './queue.js';
@@ -167,6 +167,15 @@ async function processFullCrawl(
);
await Promise.all(indexPromises);
// Post-crawl curation: classify new skills, mark duplicates, update category counts
try {
const db = getDb();
await runPostCrawlCuration(db);
} catch (error) {
console.error('[curation] Post-crawl curation failed (non-fatal):', error);
}
await job.updateProgress(100);
return {
@@ -224,6 +233,15 @@ async function processIncrementalCrawl(
);
await Promise.all(indexPromises);
// Post-crawl curation: classify new skills, mark duplicates, update category counts
try {
const db = getDb();
await runPostCrawlCuration(db);
} catch (error) {
console.error('[curation] Post-crawl curation failed (non-fatal):', error);
}
await job.updateProgress(100);
return {