feat(curation): auto-curation after crawl + quality messaging
Automate curation: - Add runPostCrawlCuration() to DB queries — runs after every full and incremental crawl (classify, dedup, category counts) - No more need for manual curate.mjs runs after each crawl Quality messaging: - Homepage stats label: "Skills" → "Curated Skills" (en/fa) - Add curation note: "deduplicated and quality-filtered from X+ repos" - Update fallback counts from 172K to 16K across all touchpoints (BetaBanner, getting-started prompts, email templates) - getting-started skill count query now uses browse-ready filter Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
import type { Job } from 'bullmq';
|
||||
import { Worker } from 'bullmq';
|
||||
import pLimit from 'p-limit';
|
||||
import { createDb, type Database, discoveredRepoQueries, awesomeListQueries, addRequestQueries, skillQueries, sql } from '@skillhub/db';
|
||||
import { createDb, type Database, discoveredRepoQueries, awesomeListQueries, addRequestQueries, skillQueries, runPostCrawlCuration, sql } from '@skillhub/db';
|
||||
import { GitHubCrawler, createCrawler } from './crawler.js';
|
||||
import type { IndexJobData, IndexJobResult } from './queue.js';
|
||||
import { setupRecurringJobs } from './queue.js';
|
||||
@@ -167,6 +167,15 @@ async function processFullCrawl(
|
||||
);
|
||||
|
||||
await Promise.all(indexPromises);
|
||||
|
||||
// Post-crawl curation: classify new skills, mark duplicates, update category counts
|
||||
try {
|
||||
const db = getDb();
|
||||
await runPostCrawlCuration(db);
|
||||
} catch (error) {
|
||||
console.error('[curation] Post-crawl curation failed (non-fatal):', error);
|
||||
}
|
||||
|
||||
await job.updateProgress(100);
|
||||
|
||||
return {
|
||||
@@ -224,6 +233,15 @@ async function processIncrementalCrawl(
|
||||
);
|
||||
|
||||
await Promise.all(indexPromises);
|
||||
|
||||
// Post-crawl curation: classify new skills, mark duplicates, update category counts
|
||||
try {
|
||||
const db = getDb();
|
||||
await runPostCrawlCuration(db);
|
||||
} catch (error) {
|
||||
console.error('[curation] Post-crawl curation failed (non-fatal):', error);
|
||||
}
|
||||
|
||||
await job.updateProgress(100);
|
||||
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user