feat(curation): data quality pipeline — Phases 1-3
Add comprehensive data curation system to clean up the 197K skill dataset and show only quality browse-ready skills to users. Phase 1 — Database exploration: - Explore scripts (explore.ts, explore.mjs, explore.sql) for analysis - Discovered: 69% duplicates, 77% aggregator/fork noise Phase 2 — Data cleanup and classification: - Schema: 6 new curation columns + 4 indexes - curate.mjs: 8-step pipeline (classify, dedup, fork detection, etc.) - Result: 197K → 60K unique → 16K browse-ready skills - Bug fix: securityStatus was computed but never stored during crawl Phase 3 — UI browse-ready filters: - browseReadyFilter applied to 17+ query functions - Homepage stats show accurate browse-ready counts - Stats API filtered (previously had no WHERE clause) - Category counts recalculated (e.g. 45K → 3.1K) - Featured skills exclude duplicates and aggregators Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
airano
214
scripts/curation/curate.sql
Normal file
214
scripts/curation/curate.sql
Normal file
@@ -0,0 +1,214 @@
|
||||
-- ============================================================
|
||||
-- Phase 2: Data Cleanup & Classification (Pure SQL version)
|
||||
-- Run inside DB container:
|
||||
-- psql -U postgres -d skillhub -f /tmp/curate.sql
|
||||
--
|
||||
-- Safe to run multiple times (idempotent).
|
||||
-- ============================================================
|
||||
|
||||
\echo '======================================================================'
|
||||
\echo ' STEP 1: COMPUTE repo_skill_count'
|
||||
\echo '======================================================================'
|
||||
|
||||
UPDATE skills s SET repo_skill_count = sub.cnt
|
||||
FROM (
|
||||
SELECT github_owner, github_repo, COUNT(*)::int AS cnt
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY github_owner, github_repo
|
||||
) sub
|
||||
WHERE s.github_owner = sub.github_owner
|
||||
AND s.github_repo = sub.github_repo
|
||||
AND s.is_blocked = false;
|
||||
|
||||
\echo ' Step 1 done. Checking counts:'
|
||||
SELECT 'repo_skill_count set' AS step,
|
||||
COUNT(*) FILTER (WHERE repo_skill_count IS NOT NULL) AS done,
|
||||
COUNT(*) FILTER (WHERE repo_skill_count IS NULL) AS remaining
|
||||
FROM skills WHERE is_blocked = false;
|
||||
|
||||
|
||||
\echo ''
|
||||
\echo '======================================================================'
|
||||
\echo ' STEP 2: MARK AGGREGATORS (repos with 50+ skills)'
|
||||
\echo '======================================================================'
|
||||
|
||||
UPDATE skills s SET skill_type = 'aggregator'
|
||||
FROM (
|
||||
SELECT github_owner, github_repo
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY github_owner, github_repo
|
||||
HAVING COUNT(*) >= 50
|
||||
) agg
|
||||
WHERE s.github_owner = agg.github_owner
|
||||
AND s.github_repo = agg.github_repo
|
||||
AND s.is_blocked = false
|
||||
AND s.skill_type IS NULL;
|
||||
|
||||
\echo ' Step 2 done.'
|
||||
|
||||
|
||||
\echo ''
|
||||
\echo '======================================================================'
|
||||
\echo ' STEP 3: CLASSIFY REMAINING REPOS'
|
||||
\echo '======================================================================'
|
||||
|
||||
-- 3a: 10-49 skills + name contains marketplace/awesome/collection/registry → aggregator
|
||||
UPDATE skills s SET skill_type = 'aggregator'
|
||||
FROM (
|
||||
SELECT github_owner, github_repo
|
||||
FROM skills WHERE is_blocked = false AND skill_type IS NULL
|
||||
GROUP BY github_owner, github_repo
|
||||
HAVING COUNT(*) >= 10
|
||||
AND (github_repo ILIKE '%marketplace%'
|
||||
OR github_repo ILIKE '%awesome%'
|
||||
OR github_repo ILIKE '%collection%'
|
||||
OR github_repo ILIKE '%registry%')
|
||||
) agg
|
||||
WHERE s.github_owner = agg.github_owner
|
||||
AND s.github_repo = agg.github_repo
|
||||
AND s.is_blocked = false
|
||||
AND s.skill_type IS NULL;
|
||||
|
||||
-- 3b: 3+ skills → collection
|
||||
UPDATE skills s SET skill_type = 'collection'
|
||||
FROM (
|
||||
SELECT github_owner, github_repo
|
||||
FROM skills WHERE is_blocked = false AND skill_type IS NULL
|
||||
GROUP BY github_owner, github_repo
|
||||
HAVING COUNT(*) >= 3
|
||||
) col
|
||||
WHERE s.github_owner = col.github_owner
|
||||
AND s.github_repo = col.github_repo
|
||||
AND s.is_blocked = false
|
||||
AND s.skill_type IS NULL;
|
||||
|
||||
-- 3c: project-bound name patterns
|
||||
UPDATE skills SET skill_type = 'project-bound'
|
||||
WHERE is_blocked = false AND skill_type IS NULL
|
||||
AND repo_skill_count <= 2
|
||||
AND (name ILIKE '%my-%' OR name ILIKE '%my\_%' ESCAPE '\'
|
||||
OR name ILIKE '%project%' OR name ILIKE '%team%' OR name ILIKE '%internal%'
|
||||
OR name ILIKE '%.mdc' OR name ILIKE '%cursorrule%'
|
||||
OR name ILIKE '%config%' OR name ILIKE '%setup%');
|
||||
|
||||
-- 3d: remaining → standalone
|
||||
UPDATE skills SET skill_type = 'standalone'
|
||||
WHERE is_blocked = false AND skill_type IS NULL;
|
||||
|
||||
\echo ' Step 3 done. Classification:'
|
||||
SELECT skill_type, COUNT(*)::int AS count
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY skill_type ORDER BY count DESC;
|
||||
|
||||
|
||||
\echo ''
|
||||
\echo '======================================================================'
|
||||
\echo ' STEP 4: FILL content_hash WITH md5 (for consistency)'
|
||||
\echo '======================================================================'
|
||||
|
||||
-- Re-hash ALL with md5 for consistent dedup
|
||||
UPDATE skills SET content_hash = md5(raw_content)
|
||||
WHERE is_blocked = false AND raw_content IS NOT NULL;
|
||||
|
||||
\echo ' Step 4 done.'
|
||||
SELECT 'content_hash' AS step,
|
||||
COUNT(*) FILTER (WHERE content_hash IS NOT NULL) AS has_hash,
|
||||
COUNT(*) FILTER (WHERE content_hash IS NULL) AS no_hash
|
||||
FROM skills WHERE is_blocked = false;
|
||||
|
||||
|
||||
\echo ''
|
||||
\echo '======================================================================'
|
||||
\echo ' STEP 5: MARK DUPLICATES BY content_hash'
|
||||
\echo '======================================================================'
|
||||
|
||||
-- First reset any previous duplicate marks (for idempotency)
|
||||
UPDATE skills SET is_duplicate = false, canonical_skill_id = NULL
|
||||
WHERE is_blocked = false AND is_duplicate = true;
|
||||
|
||||
-- Mark duplicates: keep the one with most stars (or oldest) as canonical
|
||||
WITH ranked AS (
|
||||
SELECT id, content_hash,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY content_hash
|
||||
ORDER BY github_stars DESC NULLS LAST,
|
||||
created_at ASC
|
||||
) AS rn
|
||||
FROM skills
|
||||
WHERE is_blocked = false AND content_hash IS NOT NULL
|
||||
),
|
||||
canonicals AS (
|
||||
SELECT content_hash, id AS canonical_id
|
||||
FROM ranked WHERE rn = 1
|
||||
)
|
||||
UPDATE skills s
|
||||
SET is_duplicate = true,
|
||||
canonical_skill_id = c.canonical_id
|
||||
FROM ranked r
|
||||
JOIN canonicals c ON r.content_hash = c.content_hash
|
||||
WHERE s.id = r.id
|
||||
AND r.rn > 1;
|
||||
|
||||
\echo ' Step 5 done.'
|
||||
SELECT 'dedup' AS step,
|
||||
COUNT(*) FILTER (WHERE is_duplicate = false) AS unique_skills,
|
||||
COUNT(*) FILTER (WHERE is_duplicate = true) AS duplicates
|
||||
FROM skills WHERE is_blocked = false;
|
||||
|
||||
|
||||
\echo ''
|
||||
\echo '======================================================================'
|
||||
\echo ' STEP 6: DETECT FORK MARKETPLACE REPOS'
|
||||
\echo '======================================================================'
|
||||
|
||||
-- Re-classify fork repos (same repo name in 3+ owners with 20+ skills) as aggregator
|
||||
UPDATE skills SET skill_type = 'aggregator'
|
||||
WHERE is_blocked = false
|
||||
AND repo_skill_count >= 20
|
||||
AND (skill_type IS NULL OR skill_type != 'aggregator')
|
||||
AND github_repo IN (
|
||||
SELECT github_repo
|
||||
FROM skills
|
||||
WHERE is_blocked = false AND repo_skill_count >= 20
|
||||
GROUP BY github_repo
|
||||
HAVING COUNT(DISTINCT github_owner) >= 3
|
||||
);
|
||||
|
||||
\echo ' Step 6 done. Fork patterns:'
|
||||
SELECT github_repo, COUNT(DISTINCT github_owner)::int AS owners,
|
||||
COUNT(*)::int AS total_skills
|
||||
FROM skills
|
||||
WHERE is_blocked = false AND repo_skill_count >= 20
|
||||
GROUP BY github_repo
|
||||
HAVING COUNT(DISTINCT github_owner) >= 3
|
||||
ORDER BY owners DESC
|
||||
LIMIT 15;
|
||||
|
||||
|
||||
\echo ''
|
||||
\echo '======================================================================'
|
||||
\echo ' STEP 7: FINAL SUMMARY'
|
||||
\echo '======================================================================'
|
||||
|
||||
SELECT
|
||||
COUNT(*)::int AS total,
|
||||
COUNT(*) FILTER (WHERE is_duplicate = false)::int AS unique_skills,
|
||||
COUNT(*) FILTER (WHERE is_duplicate = true)::int AS duplicates,
|
||||
COUNT(*) FILTER (WHERE skill_type = 'standalone' AND is_duplicate = false)::int AS standalone,
|
||||
COUNT(*) FILTER (WHERE skill_type = 'collection' AND is_duplicate = false)::int AS collection_type,
|
||||
COUNT(*) FILTER (WHERE skill_type = 'aggregator' AND is_duplicate = false)::int AS aggregator,
|
||||
COUNT(*) FILTER (WHERE skill_type = 'project-bound' AND is_duplicate = false)::int AS project_bound,
|
||||
COUNT(*) FILTER (WHERE skill_type IN ('standalone','collection') AND is_duplicate = false)::int AS browse_ready
|
||||
FROM skills WHERE is_blocked = false;
|
||||
|
||||
\echo ''
|
||||
\echo ' Top 20 standalone skills by stars:'
|
||||
SELECT id, github_stars AS stars, COALESCE(download_count,0) AS dl,
|
||||
LEFT(description, 70) AS description
|
||||
FROM skills
|
||||
WHERE is_blocked = false AND is_duplicate = false AND skill_type = 'standalone'
|
||||
ORDER BY github_stars DESC NULLS LAST
|
||||
LIMIT 20;
|
||||
|
||||
\echo ''
|
||||
\echo ' DONE!'
|
||||
Reference in New Issue
Block a user