feat(curation): data quality pipeline — Phases 1-3
Add comprehensive data curation system to clean up the 197K skill dataset and show only quality browse-ready skills to users. Phase 1 — Database exploration: - Explore scripts (explore.ts, explore.mjs, explore.sql) for analysis - Discovered: 69% duplicates, 77% aggregator/fork noise Phase 2 — Data cleanup and classification: - Schema: 6 new curation columns + 4 indexes - curate.mjs: 8-step pipeline (classify, dedup, fork detection, etc.) - Result: 197K → 60K unique → 16K browse-ready skills - Bug fix: securityStatus was computed but never stored during crawl Phase 3 — UI browse-ready filters: - browseReadyFilter applied to 17+ query functions - Homepage stats show accurate browse-ready counts - Stats API filtered (previously had no WHERE clause) - Category counts recalculated (e.g. 45K → 3.1K) - Featured skills exclude duplicates and aggregators Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
airano
505
scripts/curation/curate.mjs
Normal file
505
scripts/curation/curate.mjs
Normal file
@@ -0,0 +1,505 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Phase 2: Data Cleanup & Classification
|
||||
*
|
||||
* Runs all curation steps in order:
|
||||
* Step 1: Compute repo_skill_count for every skill
|
||||
* Step 2: Mark aggregators (repos with 50+ skills)
|
||||
* Step 3: Classify remaining repos (collection / standalone / project-bound)
|
||||
* Step 4: Fill missing content_hash values
|
||||
* Step 5: Mark duplicates by content_hash (canonical = highest stars or oldest)
|
||||
* Step 6: Detect fork marketplace repos
|
||||
* Step 7: Recalculate category skill counts (browse-ready only)
|
||||
* Step 8: Summary report
|
||||
*
|
||||
* Usage:
|
||||
* DATABASE_URL=postgres://... node scripts/curation/curate.mjs
|
||||
* # Or with local Docker:
|
||||
* DATABASE_URL=postgresql://postgres:postgres@localhost:5432/skillhub node scripts/curation/curate.mjs
|
||||
*
|
||||
* Options:
|
||||
* --dry-run Show what would change without writing
|
||||
* --step=N Run only step N (1-8)
|
||||
*/
|
||||
|
||||
import { createRequire } from 'module';
|
||||
import { resolve, dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
// ─── Find pg module ───
|
||||
let pg;
|
||||
const tryPaths = [
|
||||
resolve(__dirname, '../..', 'package.json'),
|
||||
'/tmp/package.json',
|
||||
process.env.APPDATA ? resolve(process.env.APPDATA, '..', 'Local', 'Temp', 'package.json') : null,
|
||||
].filter(Boolean);
|
||||
for (const p of tryPaths) {
|
||||
try { pg = createRequire(p)('pg'); break; } catch {}
|
||||
}
|
||||
if (!pg) { console.error('pg not found. Run: npm install pg'); process.exit(1); }
|
||||
|
||||
// ─── Config ───
|
||||
const DATABASE_URL = process.env.DATABASE_URL || 'postgresql://postgres:postgres@localhost:5432/skillhub';
|
||||
const DRY_RUN = process.argv.includes('--dry-run');
|
||||
const stepArg = process.argv.find(a => a.startsWith('--step='));
|
||||
const ONLY_STEP = stepArg ? parseInt(stepArg.split('=')[1]) : null;
|
||||
|
||||
// ─── Helpers ───
|
||||
function header(step, title) {
|
||||
console.log(`\n${'='.repeat(70)}\n STEP ${step}: ${title}\n${'='.repeat(70)}`);
|
||||
}
|
||||
|
||||
const n = v => Number(v ?? 0).toLocaleString('en-US');
|
||||
|
||||
// ─── Database ───
|
||||
let client;
|
||||
|
||||
async function connect() {
|
||||
client = new pg.Client({
|
||||
connectionString: DATABASE_URL,
|
||||
ssl: false,
|
||||
connectionTimeoutMillis: 15000,
|
||||
query_timeout: 600000, // 10 min for big updates
|
||||
keepAlive: true,
|
||||
});
|
||||
await client.connect();
|
||||
console.log('Connected to database');
|
||||
}
|
||||
|
||||
async function query(sql, params = []) {
|
||||
const result = await client.query(sql, params);
|
||||
return result;
|
||||
}
|
||||
|
||||
// ─── Step 1: Compute repo_skill_count ───
|
||||
async function step1_repoSkillCount() {
|
||||
header(1, 'COMPUTE repo_skill_count');
|
||||
|
||||
const countResult = await query(`
|
||||
SELECT COUNT(*)::int AS total
|
||||
FROM skills
|
||||
WHERE is_blocked = false AND repo_skill_count IS NULL
|
||||
`);
|
||||
console.log(` Skills without repo_skill_count: ${n(countResult.rows[0].total)}`);
|
||||
|
||||
if (DRY_RUN) { console.log(' [DRY RUN] Would update all skills'); return; }
|
||||
|
||||
const result = await query(`
|
||||
UPDATE skills s SET repo_skill_count = sub.cnt
|
||||
FROM (
|
||||
SELECT github_owner, github_repo, COUNT(*)::int AS cnt
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY github_owner, github_repo
|
||||
) sub
|
||||
WHERE s.github_owner = sub.github_owner
|
||||
AND s.github_repo = sub.github_repo
|
||||
AND s.is_blocked = false
|
||||
`);
|
||||
console.log(` Updated: ${n(result.rowCount)} skills`);
|
||||
}
|
||||
|
||||
// ─── Step 2: Mark aggregators ───
|
||||
async function step2_markAggregators() {
|
||||
header(2, 'MARK AGGREGATORS (repos with 50+ skills)');
|
||||
|
||||
// Preview
|
||||
const preview = await query(`
|
||||
SELECT github_owner || '/' || github_repo AS repo, COUNT(*)::int AS skills,
|
||||
MAX(github_stars)::int AS stars
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY github_owner, github_repo
|
||||
HAVING COUNT(*) >= 50
|
||||
ORDER BY skills DESC LIMIT 20
|
||||
`);
|
||||
console.log(` Top aggregator repos (${preview.rowCount} total):`);
|
||||
for (const r of preview.rows.slice(0, 10)) {
|
||||
console.log(` ${r.repo}: ${n(r.skills)} skills (${n(r.stars)} stars)`);
|
||||
}
|
||||
|
||||
if (DRY_RUN) { console.log(' [DRY RUN] Would mark skills in these repos'); return; }
|
||||
|
||||
const result = await query(`
|
||||
UPDATE skills s SET skill_type = 'aggregator'
|
||||
FROM (
|
||||
SELECT github_owner, github_repo
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY github_owner, github_repo
|
||||
HAVING COUNT(*) >= 50
|
||||
) agg
|
||||
WHERE s.github_owner = agg.github_owner
|
||||
AND s.github_repo = agg.github_repo
|
||||
AND s.is_blocked = false
|
||||
AND s.skill_type IS NULL
|
||||
`);
|
||||
console.log(` Marked as aggregator: ${n(result.rowCount)} skills`);
|
||||
}
|
||||
|
||||
// ─── Step 3: Classify remaining repos ───
|
||||
async function step3_classifyRemaining() {
|
||||
header(3, 'CLASSIFY REMAINING REPOS');
|
||||
|
||||
// 3a: repos with 10-49 skills, name contains marketplace/awesome/collection → aggregator
|
||||
if (!DRY_RUN) {
|
||||
const aggByName = await query(`
|
||||
UPDATE skills s SET skill_type = 'aggregator'
|
||||
FROM (
|
||||
SELECT github_owner, github_repo
|
||||
FROM skills WHERE is_blocked = false AND skill_type IS NULL
|
||||
GROUP BY github_owner, github_repo
|
||||
HAVING COUNT(*) >= 10
|
||||
AND (github_repo ILIKE '%marketplace%'
|
||||
OR github_repo ILIKE '%awesome%'
|
||||
OR github_repo ILIKE '%collection%'
|
||||
OR github_repo ILIKE '%registry%')
|
||||
) agg
|
||||
WHERE s.github_owner = agg.github_owner
|
||||
AND s.github_repo = agg.github_repo
|
||||
AND s.is_blocked = false
|
||||
AND s.skill_type IS NULL
|
||||
`);
|
||||
console.log(` Aggregator by name (10+ & marketplace/awesome/registry): ${n(aggByName.rowCount)}`);
|
||||
}
|
||||
|
||||
// 3b: repos with 3-49 skills → collection
|
||||
if (!DRY_RUN) {
|
||||
const collections = await query(`
|
||||
UPDATE skills s SET skill_type = 'collection'
|
||||
FROM (
|
||||
SELECT github_owner, github_repo
|
||||
FROM skills WHERE is_blocked = false AND skill_type IS NULL
|
||||
GROUP BY github_owner, github_repo
|
||||
HAVING COUNT(*) >= 3
|
||||
) col
|
||||
WHERE s.github_owner = col.github_owner
|
||||
AND s.github_repo = col.github_repo
|
||||
AND s.is_blocked = false
|
||||
AND s.skill_type IS NULL
|
||||
`);
|
||||
console.log(` Collection (3+ skills in repo): ${n(collections.rowCount)}`);
|
||||
}
|
||||
|
||||
// 3c: single/two-skill repos with project-bound name patterns → project-bound
|
||||
if (!DRY_RUN) {
|
||||
const projectBound = await query(`
|
||||
UPDATE skills SET skill_type = 'project-bound'
|
||||
WHERE is_blocked = false AND skill_type IS NULL
|
||||
AND repo_skill_count <= 2
|
||||
AND (name ILIKE '%my-%' OR name ILIKE '%my\\_%' ESCAPE '\\'
|
||||
OR name ILIKE '%project%' OR name ILIKE '%team%' OR name ILIKE '%internal%'
|
||||
OR name ILIKE '%.mdc' OR name ILIKE '%cursorrule%'
|
||||
OR name ILIKE '%config%' OR name ILIKE '%setup%')
|
||||
`);
|
||||
console.log(` Project-bound (name pattern): ${n(projectBound.rowCount)}`);
|
||||
}
|
||||
|
||||
// 3d: remaining → standalone
|
||||
if (!DRY_RUN) {
|
||||
const standalone = await query(`
|
||||
UPDATE skills SET skill_type = 'standalone'
|
||||
WHERE is_blocked = false AND skill_type IS NULL
|
||||
`);
|
||||
console.log(` Standalone (remaining): ${n(standalone.rowCount)}`);
|
||||
}
|
||||
|
||||
// Summary
|
||||
const summary = await query(`
|
||||
SELECT skill_type, COUNT(*)::int AS count
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY skill_type ORDER BY count DESC
|
||||
`);
|
||||
console.log('\n Classification summary:');
|
||||
for (const r of summary.rows) {
|
||||
console.log(` ${(r.skill_type || 'null').padEnd(15)} ${n(r.count)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Step 4: Fill missing content_hash ───
|
||||
async function step4_fillContentHash() {
|
||||
header(4, 'FILL MISSING content_hash');
|
||||
|
||||
const countResult = await query(`
|
||||
SELECT COUNT(*)::int AS total
|
||||
FROM skills WHERE is_blocked = false AND content_hash IS NULL AND raw_content IS NOT NULL
|
||||
`);
|
||||
const missing = countResult.rows[0].total;
|
||||
console.log(` Skills missing content_hash: ${n(missing)}`);
|
||||
|
||||
if (missing === 0) { console.log(' Nothing to do'); return; }
|
||||
if (DRY_RUN) { console.log(' [DRY RUN] Would compute hash for these skills'); return; }
|
||||
|
||||
// Use md5 as a reliable hash function available in PostgreSQL
|
||||
const result = await query(`
|
||||
UPDATE skills SET content_hash = md5(raw_content)
|
||||
WHERE is_blocked = false AND content_hash IS NULL AND raw_content IS NOT NULL
|
||||
`);
|
||||
console.log(` Computed content_hash (md5) for: ${n(result.rowCount)} skills`);
|
||||
|
||||
// Note: existing hashes use a JS numeric hash (analyzer.ts hashContent).
|
||||
// We now use md5 for missing ones. For dedup purposes, same content → same hash
|
||||
// regardless of algorithm, since we compare within hash groups.
|
||||
// But mixed algorithms mean same content could have different hashes.
|
||||
// Solution: re-hash ALL with md5 for consistency.
|
||||
console.log(' Re-hashing ALL skills with md5 for consistency...');
|
||||
const rehash = await query(`
|
||||
UPDATE skills SET content_hash = md5(raw_content)
|
||||
WHERE is_blocked = false AND raw_content IS NOT NULL
|
||||
`);
|
||||
console.log(` Re-hashed: ${n(rehash.rowCount)} skills`);
|
||||
}
|
||||
|
||||
// ─── Step 5: Mark duplicates ───
|
||||
async function step5_markDuplicates() {
|
||||
header(5, 'MARK DUPLICATES BY content_hash');
|
||||
|
||||
// Find duplicate groups
|
||||
const dupGroups = await query(`
|
||||
SELECT content_hash, COUNT(*)::int AS copies
|
||||
FROM skills
|
||||
WHERE is_blocked = false AND content_hash IS NOT NULL AND is_duplicate = false
|
||||
GROUP BY content_hash
|
||||
HAVING COUNT(*) > 1
|
||||
ORDER BY copies DESC
|
||||
`);
|
||||
const totalDupGroups = dupGroups.rowCount;
|
||||
const totalDupSkills = dupGroups.rows.reduce((sum, r) => sum + r.copies, 0);
|
||||
const removable = dupGroups.rows.reduce((sum, r) => sum + r.copies - 1, 0);
|
||||
|
||||
console.log(` Duplicate groups: ${n(totalDupGroups)}`);
|
||||
console.log(` Total skills in dup groups: ${n(totalDupSkills)}`);
|
||||
console.log(` Removable (keeping 1 per group): ${n(removable)}`);
|
||||
console.log(` Top 5 by copies:`);
|
||||
for (const r of dupGroups.rows.slice(0, 5)) {
|
||||
console.log(` hash=${r.content_hash}: ${r.copies} copies`);
|
||||
}
|
||||
|
||||
if (DRY_RUN) { console.log(' [DRY RUN] Would mark duplicates'); return; }
|
||||
|
||||
// For each duplicate group:
|
||||
// - canonical = highest github_stars, then oldest created_at
|
||||
// - rest = is_duplicate = true, canonical_skill_id = canonical.id
|
||||
const markResult = await query(`
|
||||
WITH ranked AS (
|
||||
SELECT id, content_hash,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY content_hash
|
||||
ORDER BY github_stars DESC NULLS LAST,
|
||||
created_at ASC
|
||||
) AS rn
|
||||
FROM skills
|
||||
WHERE is_blocked = false AND content_hash IS NOT NULL AND is_duplicate = false
|
||||
),
|
||||
canonicals AS (
|
||||
SELECT content_hash, id AS canonical_id
|
||||
FROM ranked WHERE rn = 1
|
||||
)
|
||||
UPDATE skills s
|
||||
SET is_duplicate = true,
|
||||
canonical_skill_id = c.canonical_id
|
||||
FROM ranked r
|
||||
JOIN canonicals c ON r.content_hash = c.content_hash
|
||||
WHERE s.id = r.id
|
||||
AND r.rn > 1
|
||||
`);
|
||||
console.log(` Marked as duplicate: ${n(markResult.rowCount)} skills`);
|
||||
|
||||
// Verify
|
||||
const verify = await query(`
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE is_duplicate = false)::int AS unique_skills,
|
||||
COUNT(*) FILTER (WHERE is_duplicate = true)::int AS duplicates
|
||||
FROM skills WHERE is_blocked = false
|
||||
`);
|
||||
console.log(` After dedup: ${n(verify.rows[0].unique_skills)} unique, ${n(verify.rows[0].duplicates)} duplicates`);
|
||||
}
|
||||
|
||||
// ─── Step 6: Detect fork marketplace repos ───
|
||||
async function step6_detectForks() {
|
||||
header(6, 'DETECT FORK MARKETPLACE REPOS');
|
||||
|
||||
// Known fork patterns: repos with same name across many owners
|
||||
const forkPatterns = await query(`
|
||||
SELECT github_repo, COUNT(DISTINCT github_owner)::int AS owners,
|
||||
COUNT(*)::int AS total_skills
|
||||
FROM skills
|
||||
WHERE is_blocked = false
|
||||
AND repo_skill_count >= 20
|
||||
GROUP BY github_repo
|
||||
HAVING COUNT(DISTINCT github_owner) >= 3
|
||||
ORDER BY owners DESC
|
||||
`);
|
||||
|
||||
console.log(` Repo names appearing in 3+ owners (with 20+ skills each):`);
|
||||
for (const r of forkPatterns.rows) {
|
||||
console.log(` ${r.github_repo}: ${r.owners} owners, ${n(r.total_skills)} skills`);
|
||||
}
|
||||
|
||||
if (DRY_RUN) { console.log(' [DRY RUN] Would mark fork repo skills'); return; }
|
||||
|
||||
// For fork repos, all copies should be checked as duplicates
|
||||
// The canonical is already handled by step 5 (content_hash dedup)
|
||||
// Here we just ensure they're typed correctly as aggregator
|
||||
if (forkPatterns.rows.length > 0) {
|
||||
const repoNames = forkPatterns.rows.map(r => r.github_repo);
|
||||
const placeholders = repoNames.map((_, i) => `$${i + 1}`).join(',');
|
||||
const markForks = await query(`
|
||||
UPDATE skills SET skill_type = 'aggregator'
|
||||
WHERE is_blocked = false
|
||||
AND github_repo IN (${placeholders})
|
||||
AND repo_skill_count >= 20
|
||||
AND (skill_type IS NULL OR skill_type != 'aggregator')
|
||||
`, repoNames);
|
||||
console.log(` Re-classified as aggregator (fork repos): ${n(markForks.rowCount)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Step 7: Recalculate category counts ───
|
||||
async function step7_categoryCounts() {
|
||||
header(7, 'RECALCULATE CATEGORY COUNTS');
|
||||
|
||||
// Update skill_count on each category to only count browse-ready skills
|
||||
const result = await query(`
|
||||
UPDATE categories c
|
||||
SET skill_count = sub.cnt
|
||||
FROM (
|
||||
SELECT sc.category_id, COUNT(*)::int AS cnt
|
||||
FROM skill_categories sc
|
||||
JOIN skills s ON sc.skill_id = s.id
|
||||
WHERE s.is_blocked = false
|
||||
AND s.is_duplicate = false
|
||||
AND (s.skill_type IS NULL OR s.skill_type != 'aggregator')
|
||||
GROUP BY sc.category_id
|
||||
) sub
|
||||
WHERE c.id = sub.category_id
|
||||
AND c.skill_count IS DISTINCT FROM sub.cnt
|
||||
`);
|
||||
console.log(` Updated ${result.rowCount} category counts (browse-ready only)`);
|
||||
|
||||
// Also zero out categories that have no browse-ready skills
|
||||
const zeroResult = await query(`
|
||||
UPDATE categories c
|
||||
SET skill_count = 0
|
||||
WHERE c.skill_count > 0
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM skill_categories sc
|
||||
JOIN skills s ON sc.skill_id = s.id
|
||||
WHERE sc.category_id = c.id
|
||||
AND s.is_blocked = false
|
||||
AND s.is_duplicate = false
|
||||
AND (s.skill_type IS NULL OR s.skill_type != 'aggregator')
|
||||
)
|
||||
`);
|
||||
if (zeroResult.rowCount > 0) {
|
||||
console.log(` Zeroed ${zeroResult.rowCount} categories with no browse-ready skills`);
|
||||
}
|
||||
|
||||
// Show category counts
|
||||
const cats = await query(`
|
||||
SELECT name, skill_count FROM categories
|
||||
WHERE id NOT LIKE 'parent-%'
|
||||
ORDER BY skill_count DESC
|
||||
LIMIT 10
|
||||
`);
|
||||
console.log(' Top 10 categories by browse-ready count:');
|
||||
for (const r of cats.rows) {
|
||||
console.log(` ${n(r.skill_count).padStart(6)} ${r.name}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Step 8: Summary ───
|
||||
async function step8_summary() {
|
||||
header(8, 'FINAL SUMMARY');
|
||||
|
||||
const summary = await query(`
|
||||
SELECT
|
||||
COUNT(*)::int AS total,
|
||||
COUNT(*) FILTER (WHERE is_duplicate = false)::int AS unique_skills,
|
||||
COUNT(*) FILTER (WHERE is_duplicate = true)::int AS duplicates,
|
||||
COUNT(*) FILTER (WHERE skill_type = 'standalone' AND is_duplicate = false)::int AS standalone,
|
||||
COUNT(*) FILTER (WHERE skill_type = 'collection' AND is_duplicate = false)::int AS collection,
|
||||
COUNT(*) FILTER (WHERE skill_type = 'aggregator' AND is_duplicate = false)::int AS aggregator,
|
||||
COUNT(*) FILTER (WHERE skill_type = 'project-bound' AND is_duplicate = false)::int AS project_bound,
|
||||
COUNT(*) FILTER (WHERE skill_type IS NULL AND is_duplicate = false)::int AS unclassified
|
||||
FROM skills WHERE is_blocked = false
|
||||
`);
|
||||
const s = summary.rows[0];
|
||||
|
||||
console.log(`
|
||||
┌─────────────────────────────────────────────────────┐
|
||||
│ CURATION SUMMARY │
|
||||
├─────────────────────────────────────────────────────┤
|
||||
│ Total skills: ${n(s.total).padStart(8)} │
|
||||
│ Unique skills: ${n(s.unique_skills).padStart(8)} │
|
||||
│ Duplicates: ${n(s.duplicates).padStart(8)} │
|
||||
├─────────────────────────────────────────────────────┤
|
||||
│ Unique by type: │
|
||||
│ Standalone: ${n(s.standalone).padStart(8)} │
|
||||
│ Collection: ${n(s.collection).padStart(8)} │
|
||||
│ Aggregator: ${n(s.aggregator).padStart(8)} │
|
||||
│ Project-bound: ${n(s.project_bound).padStart(8)} │
|
||||
│ Unclassified: ${n(s.unclassified).padStart(8)} │
|
||||
└─────────────────────────────────────────────────────┘
|
||||
`);
|
||||
|
||||
// Interesting standalone skills
|
||||
const topStandalone = await query(`
|
||||
SELECT id, name, github_stars AS stars, COALESCE(download_count,0) AS dl,
|
||||
LEFT(description, 70) AS description
|
||||
FROM skills
|
||||
WHERE is_blocked = false AND is_duplicate = false
|
||||
AND skill_type = 'standalone'
|
||||
ORDER BY github_stars DESC NULLS LAST
|
||||
LIMIT 20
|
||||
`);
|
||||
console.log(' Top 20 standalone skills by stars:');
|
||||
for (const r of topStandalone.rows) {
|
||||
console.log(` [${n(r.stars)}★ ${n(r.dl)}↓] ${r.id}`);
|
||||
console.log(` ${r.description}`);
|
||||
}
|
||||
|
||||
// Browse-ready count (what users would see)
|
||||
const browseReady = await query(`
|
||||
SELECT COUNT(*)::int AS count
|
||||
FROM skills
|
||||
WHERE is_blocked = false
|
||||
AND is_duplicate = false
|
||||
AND skill_type IN ('standalone', 'collection')
|
||||
`);
|
||||
console.log(`\n Browse-ready skills (standalone + collection, unique): ${n(browseReady.rows[0].count)}`);
|
||||
}
|
||||
|
||||
// ─── Main ───
|
||||
async function main() {
|
||||
const t0 = Date.now();
|
||||
await connect();
|
||||
|
||||
if (DRY_RUN) console.log('\n *** DRY RUN MODE — no changes will be made ***\n');
|
||||
|
||||
const steps = [
|
||||
step1_repoSkillCount,
|
||||
step2_markAggregators,
|
||||
step3_classifyRemaining,
|
||||
step4_fillContentHash,
|
||||
step5_markDuplicates,
|
||||
step6_detectForks,
|
||||
step7_categoryCounts,
|
||||
step8_summary,
|
||||
];
|
||||
|
||||
for (let i = 0; i < steps.length; i++) {
|
||||
if (ONLY_STEP && ONLY_STEP !== i + 1) continue;
|
||||
await steps[i]();
|
||||
}
|
||||
|
||||
const dur = ((Date.now() - t0) / 1000).toFixed(1);
|
||||
console.log(`\nCompleted in ${dur}s`);
|
||||
|
||||
await client.end().catch(() => {});
|
||||
}
|
||||
|
||||
main().catch(async e => {
|
||||
console.error('FAILED:', e.message || e);
|
||||
await client?.end().catch(() => {});
|
||||
process.exit(1);
|
||||
});
|
||||
214
scripts/curation/curate.sql
Normal file
214
scripts/curation/curate.sql
Normal file
@@ -0,0 +1,214 @@
|
||||
-- ============================================================
|
||||
-- Phase 2: Data Cleanup & Classification (Pure SQL version)
|
||||
-- Run inside DB container:
|
||||
-- psql -U postgres -d skillhub -f /tmp/curate.sql
|
||||
--
|
||||
-- Safe to run multiple times (idempotent).
|
||||
-- ============================================================
|
||||
|
||||
\echo '======================================================================'
|
||||
\echo ' STEP 1: COMPUTE repo_skill_count'
|
||||
\echo '======================================================================'
|
||||
|
||||
UPDATE skills s SET repo_skill_count = sub.cnt
|
||||
FROM (
|
||||
SELECT github_owner, github_repo, COUNT(*)::int AS cnt
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY github_owner, github_repo
|
||||
) sub
|
||||
WHERE s.github_owner = sub.github_owner
|
||||
AND s.github_repo = sub.github_repo
|
||||
AND s.is_blocked = false;
|
||||
|
||||
\echo ' Step 1 done. Checking counts:'
|
||||
SELECT 'repo_skill_count set' AS step,
|
||||
COUNT(*) FILTER (WHERE repo_skill_count IS NOT NULL) AS done,
|
||||
COUNT(*) FILTER (WHERE repo_skill_count IS NULL) AS remaining
|
||||
FROM skills WHERE is_blocked = false;
|
||||
|
||||
|
||||
\echo ''
|
||||
\echo '======================================================================'
|
||||
\echo ' STEP 2: MARK AGGREGATORS (repos with 50+ skills)'
|
||||
\echo '======================================================================'
|
||||
|
||||
UPDATE skills s SET skill_type = 'aggregator'
|
||||
FROM (
|
||||
SELECT github_owner, github_repo
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY github_owner, github_repo
|
||||
HAVING COUNT(*) >= 50
|
||||
) agg
|
||||
WHERE s.github_owner = agg.github_owner
|
||||
AND s.github_repo = agg.github_repo
|
||||
AND s.is_blocked = false
|
||||
AND s.skill_type IS NULL;
|
||||
|
||||
\echo ' Step 2 done.'
|
||||
|
||||
|
||||
\echo ''
|
||||
\echo '======================================================================'
|
||||
\echo ' STEP 3: CLASSIFY REMAINING REPOS'
|
||||
\echo '======================================================================'
|
||||
|
||||
-- 3a: 10-49 skills + name contains marketplace/awesome/collection/registry → aggregator
|
||||
UPDATE skills s SET skill_type = 'aggregator'
|
||||
FROM (
|
||||
SELECT github_owner, github_repo
|
||||
FROM skills WHERE is_blocked = false AND skill_type IS NULL
|
||||
GROUP BY github_owner, github_repo
|
||||
HAVING COUNT(*) >= 10
|
||||
AND (github_repo ILIKE '%marketplace%'
|
||||
OR github_repo ILIKE '%awesome%'
|
||||
OR github_repo ILIKE '%collection%'
|
||||
OR github_repo ILIKE '%registry%')
|
||||
) agg
|
||||
WHERE s.github_owner = agg.github_owner
|
||||
AND s.github_repo = agg.github_repo
|
||||
AND s.is_blocked = false
|
||||
AND s.skill_type IS NULL;
|
||||
|
||||
-- 3b: 3+ skills → collection
|
||||
UPDATE skills s SET skill_type = 'collection'
|
||||
FROM (
|
||||
SELECT github_owner, github_repo
|
||||
FROM skills WHERE is_blocked = false AND skill_type IS NULL
|
||||
GROUP BY github_owner, github_repo
|
||||
HAVING COUNT(*) >= 3
|
||||
) col
|
||||
WHERE s.github_owner = col.github_owner
|
||||
AND s.github_repo = col.github_repo
|
||||
AND s.is_blocked = false
|
||||
AND s.skill_type IS NULL;
|
||||
|
||||
-- 3c: project-bound name patterns
|
||||
UPDATE skills SET skill_type = 'project-bound'
|
||||
WHERE is_blocked = false AND skill_type IS NULL
|
||||
AND repo_skill_count <= 2
|
||||
AND (name ILIKE '%my-%' OR name ILIKE '%my\_%' ESCAPE '\'
|
||||
OR name ILIKE '%project%' OR name ILIKE '%team%' OR name ILIKE '%internal%'
|
||||
OR name ILIKE '%.mdc' OR name ILIKE '%cursorrule%'
|
||||
OR name ILIKE '%config%' OR name ILIKE '%setup%');
|
||||
|
||||
-- 3d: remaining → standalone
|
||||
UPDATE skills SET skill_type = 'standalone'
|
||||
WHERE is_blocked = false AND skill_type IS NULL;
|
||||
|
||||
\echo ' Step 3 done. Classification:'
|
||||
SELECT skill_type, COUNT(*)::int AS count
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY skill_type ORDER BY count DESC;
|
||||
|
||||
|
||||
\echo ''
|
||||
\echo '======================================================================'
|
||||
\echo ' STEP 4: FILL content_hash WITH md5 (for consistency)'
|
||||
\echo '======================================================================'
|
||||
|
||||
-- Re-hash ALL with md5 for consistent dedup
|
||||
UPDATE skills SET content_hash = md5(raw_content)
|
||||
WHERE is_blocked = false AND raw_content IS NOT NULL;
|
||||
|
||||
\echo ' Step 4 done.'
|
||||
SELECT 'content_hash' AS step,
|
||||
COUNT(*) FILTER (WHERE content_hash IS NOT NULL) AS has_hash,
|
||||
COUNT(*) FILTER (WHERE content_hash IS NULL) AS no_hash
|
||||
FROM skills WHERE is_blocked = false;
|
||||
|
||||
|
||||
\echo ''
|
||||
\echo '======================================================================'
|
||||
\echo ' STEP 5: MARK DUPLICATES BY content_hash'
|
||||
\echo '======================================================================'
|
||||
|
||||
-- First reset any previous duplicate marks (for idempotency)
|
||||
UPDATE skills SET is_duplicate = false, canonical_skill_id = NULL
|
||||
WHERE is_blocked = false AND is_duplicate = true;
|
||||
|
||||
-- Mark duplicates: keep the one with most stars (or oldest) as canonical
|
||||
WITH ranked AS (
|
||||
SELECT id, content_hash,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY content_hash
|
||||
ORDER BY github_stars DESC NULLS LAST,
|
||||
created_at ASC
|
||||
) AS rn
|
||||
FROM skills
|
||||
WHERE is_blocked = false AND content_hash IS NOT NULL
|
||||
),
|
||||
canonicals AS (
|
||||
SELECT content_hash, id AS canonical_id
|
||||
FROM ranked WHERE rn = 1
|
||||
)
|
||||
UPDATE skills s
|
||||
SET is_duplicate = true,
|
||||
canonical_skill_id = c.canonical_id
|
||||
FROM ranked r
|
||||
JOIN canonicals c ON r.content_hash = c.content_hash
|
||||
WHERE s.id = r.id
|
||||
AND r.rn > 1;
|
||||
|
||||
\echo ' Step 5 done.'
|
||||
SELECT 'dedup' AS step,
|
||||
COUNT(*) FILTER (WHERE is_duplicate = false) AS unique_skills,
|
||||
COUNT(*) FILTER (WHERE is_duplicate = true) AS duplicates
|
||||
FROM skills WHERE is_blocked = false;
|
||||
|
||||
|
||||
\echo ''
|
||||
\echo '======================================================================'
|
||||
\echo ' STEP 6: DETECT FORK MARKETPLACE REPOS'
|
||||
\echo '======================================================================'
|
||||
|
||||
-- Re-classify fork repos (same repo name in 3+ owners with 20+ skills) as aggregator
|
||||
UPDATE skills SET skill_type = 'aggregator'
|
||||
WHERE is_blocked = false
|
||||
AND repo_skill_count >= 20
|
||||
AND (skill_type IS NULL OR skill_type != 'aggregator')
|
||||
AND github_repo IN (
|
||||
SELECT github_repo
|
||||
FROM skills
|
||||
WHERE is_blocked = false AND repo_skill_count >= 20
|
||||
GROUP BY github_repo
|
||||
HAVING COUNT(DISTINCT github_owner) >= 3
|
||||
);
|
||||
|
||||
\echo ' Step 6 done. Fork patterns:'
|
||||
SELECT github_repo, COUNT(DISTINCT github_owner)::int AS owners,
|
||||
COUNT(*)::int AS total_skills
|
||||
FROM skills
|
||||
WHERE is_blocked = false AND repo_skill_count >= 20
|
||||
GROUP BY github_repo
|
||||
HAVING COUNT(DISTINCT github_owner) >= 3
|
||||
ORDER BY owners DESC
|
||||
LIMIT 15;
|
||||
|
||||
|
||||
\echo ''
|
||||
\echo '======================================================================'
|
||||
\echo ' STEP 7: FINAL SUMMARY'
|
||||
\echo '======================================================================'
|
||||
|
||||
SELECT
|
||||
COUNT(*)::int AS total,
|
||||
COUNT(*) FILTER (WHERE is_duplicate = false)::int AS unique_skills,
|
||||
COUNT(*) FILTER (WHERE is_duplicate = true)::int AS duplicates,
|
||||
COUNT(*) FILTER (WHERE skill_type = 'standalone' AND is_duplicate = false)::int AS standalone,
|
||||
COUNT(*) FILTER (WHERE skill_type = 'collection' AND is_duplicate = false)::int AS collection_type,
|
||||
COUNT(*) FILTER (WHERE skill_type = 'aggregator' AND is_duplicate = false)::int AS aggregator,
|
||||
COUNT(*) FILTER (WHERE skill_type = 'project-bound' AND is_duplicate = false)::int AS project_bound,
|
||||
COUNT(*) FILTER (WHERE skill_type IN ('standalone','collection') AND is_duplicate = false)::int AS browse_ready
|
||||
FROM skills WHERE is_blocked = false;
|
||||
|
||||
\echo ''
|
||||
\echo ' Top 20 standalone skills by stars:'
|
||||
SELECT id, github_stars AS stars, COALESCE(download_count,0) AS dl,
|
||||
LEFT(description, 70) AS description
|
||||
FROM skills
|
||||
WHERE is_blocked = false AND is_duplicate = false AND skill_type = 'standalone'
|
||||
ORDER BY github_stars DESC NULLS LAST
|
||||
LIMIT 20;
|
||||
|
||||
\echo ''
|
||||
\echo ' DONE!'
|
||||
652
scripts/curation/explore.mjs
Normal file
652
scripts/curation/explore.mjs
Normal file
@@ -0,0 +1,652 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Phase 1: Database Exploration — Single-Query Version
|
||||
*
|
||||
* Runs ALL analytics in ONE SQL query to handle unstable connections.
|
||||
*
|
||||
* Usage:
|
||||
* PGSSLMODE=disable DATABASE_URL=postgres://... node scripts/curation/explore.mjs
|
||||
*/
|
||||
|
||||
import { createRequire } from 'module';
|
||||
import { writeFileSync } from 'fs';
|
||||
import { resolve, dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const projectRoot = resolve(__dirname, '../..');
|
||||
|
||||
let pg;
|
||||
const tryPaths = [
|
||||
'/tmp/package.json',
|
||||
process.env.APPDATA ? resolve(process.env.APPDATA, '..', 'Local', 'Temp', 'package.json') : null,
|
||||
resolve(projectRoot, 'package.json'),
|
||||
].filter(Boolean);
|
||||
for (const p of tryPaths) {
|
||||
try { pg = createRequire(p)('pg'); break; } catch {}
|
||||
}
|
||||
if (!pg) { console.error('pg not found. Run: cd /tmp && npm install pg'); process.exit(1); }
|
||||
|
||||
const DATABASE_URL = process.env.DATABASE_URL || 'postgresql://postgres:postgres@localhost:5432/skillhub';
|
||||
|
||||
process.on('uncaughtException', e => {
|
||||
if (e.code === 'ECONNRESET') return;
|
||||
console.error('Fatal:', e); process.exit(1);
|
||||
});
|
||||
|
||||
// ─── Mega Query ──────────────────────────────────────────
|
||||
// All analytics combined into one JSON result
|
||||
|
||||
const MEGA_SQL = `
|
||||
WITH active AS (
|
||||
SELECT * FROM skills WHERE is_blocked = false
|
||||
),
|
||||
|
||||
-- 1. Overall
|
||||
overall AS (
|
||||
SELECT json_build_object(
|
||||
'total', (SELECT COUNT(*)::int FROM skills),
|
||||
'active', (SELECT COUNT(*)::int FROM active),
|
||||
'blocked', (SELECT COUNT(*) FILTER (WHERE is_blocked)::int FROM skills),
|
||||
'owners', (SELECT COUNT(DISTINCT github_owner)::int FROM active),
|
||||
'repos', (SELECT COUNT(DISTINCT github_owner||'/'||github_repo)::int FROM active),
|
||||
'skillmd', (SELECT COUNT(*) FILTER (WHERE source_format='skill.md')::int FROM active),
|
||||
'other_fmt', (SELECT COUNT(*) FILTER (WHERE source_format!='skill.md')::int FROM active),
|
||||
'downloads', (SELECT COALESCE(SUM(download_count),0)::bigint FROM active),
|
||||
'views', (SELECT COALESCE(SUM(view_count),0)::bigint FROM active),
|
||||
'rated', (SELECT COUNT(*) FILTER (WHERE rating_count>0)::int FROM active),
|
||||
'first', (SELECT MIN(created_at)::text FROM active),
|
||||
'last', (SELECT MAX(created_at)::text FROM active)
|
||||
) AS data
|
||||
),
|
||||
|
||||
-- 1b. Source formats
|
||||
source_formats AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT COALESCE(source_format,'null') AS format, COUNT(*)::int AS count,
|
||||
COUNT(*) FILTER (WHERE is_blocked=false)::int AS active
|
||||
FROM skills GROUP BY source_format ORDER BY count DESC
|
||||
) t
|
||||
),
|
||||
|
||||
-- 2. Stars distribution
|
||||
stars_dist AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT CASE
|
||||
WHEN github_stars IS NULL OR github_stars=0 THEN '0'
|
||||
WHEN github_stars BETWEEN 1 AND 5 THEN '1-5'
|
||||
WHEN github_stars BETWEEN 6 AND 10 THEN '6-10'
|
||||
WHEN github_stars BETWEEN 11 AND 50 THEN '11-50'
|
||||
WHEN github_stars BETWEEN 51 AND 100 THEN '51-100'
|
||||
WHEN github_stars BETWEEN 101 AND 500 THEN '101-500'
|
||||
WHEN github_stars BETWEEN 501 AND 1000 THEN '501-1K'
|
||||
WHEN github_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
|
||||
WHEN github_stars BETWEEN 5001 AND 10000 THEN '5K-10K'
|
||||
WHEN github_stars BETWEEN 10001 AND 50000 THEN '10K-50K'
|
||||
ELSE '50K+'
|
||||
END AS stars, COUNT(*)::int AS skills,
|
||||
COUNT(DISTINCT github_owner)::int AS owners,
|
||||
COUNT(DISTINCT github_owner||'/'||github_repo)::int AS repos
|
||||
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(github_stars,0))
|
||||
) t
|
||||
),
|
||||
|
||||
-- 3. Downloads
|
||||
dl_dist AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT CASE
|
||||
WHEN COALESCE(download_count,0)=0 THEN '0'
|
||||
WHEN download_count BETWEEN 1 AND 5 THEN '1-5'
|
||||
WHEN download_count BETWEEN 6 AND 50 THEN '6-50'
|
||||
WHEN download_count BETWEEN 51 AND 500 THEN '51-500'
|
||||
ELSE '500+'
|
||||
END AS range, COUNT(*)::int AS skills
|
||||
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(download_count,0))
|
||||
) t
|
||||
),
|
||||
|
||||
-- 3b. Views
|
||||
vw_dist AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT CASE
|
||||
WHEN COALESCE(view_count,0)=0 THEN '0'
|
||||
WHEN view_count BETWEEN 1 AND 10 THEN '1-10'
|
||||
WHEN view_count BETWEEN 11 AND 100 THEN '11-100'
|
||||
WHEN view_count BETWEEN 101 AND 1000 THEN '101-1K'
|
||||
ELSE '1K+'
|
||||
END AS range, COUNT(*)::int AS skills
|
||||
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(view_count,0))
|
||||
) t
|
||||
),
|
||||
|
||||
-- 3c. Top downloaded
|
||||
top_dl AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT id, name, download_count AS dl, view_count AS views,
|
||||
github_stars AS stars, github_owner AS owner
|
||||
FROM active WHERE COALESCE(download_count,0)>0
|
||||
ORDER BY download_count DESC LIMIT 20
|
||||
) t
|
||||
),
|
||||
|
||||
-- 3d. Top viewed
|
||||
top_vw AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT id, name, view_count AS views, download_count AS dl,
|
||||
github_stars AS stars, github_owner AS owner
|
||||
FROM active WHERE COALESCE(view_count,0)>0
|
||||
ORDER BY view_count DESC LIMIT 20
|
||||
) t
|
||||
),
|
||||
|
||||
-- 4. Security
|
||||
security AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT COALESCE(security_status,'null') AS status, COUNT(*)::int AS count
|
||||
FROM active GROUP BY security_status ORDER BY count DESC
|
||||
) t
|
||||
),
|
||||
|
||||
-- 5. Content stats
|
||||
content_stats AS (
|
||||
SELECT json_build_object(
|
||||
'has_content', COUNT(*) FILTER (WHERE raw_content IS NOT NULL)::int,
|
||||
'no_content', COUNT(*) FILTER (WHERE raw_content IS NULL)::int,
|
||||
'avg_len', ROUND(AVG(LENGTH(raw_content)) FILTER (WHERE raw_content IS NOT NULL))::int,
|
||||
'max_len', MAX(LENGTH(raw_content))::int,
|
||||
'median_len', PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY LENGTH(raw_content))
|
||||
FILTER (WHERE raw_content IS NOT NULL)::int,
|
||||
'avg_desc', ROUND(AVG(LENGTH(description)))::int,
|
||||
'good_desc', COUNT(*) FILTER (WHERE LENGTH(description)>100)::int,
|
||||
'short_desc', COUNT(*) FILTER (WHERE LENGTH(description)<=20)::int,
|
||||
'has_ver', COUNT(*) FILTER (WHERE version IS NOT NULL)::int,
|
||||
'has_lic', COUNT(*) FILTER (WHERE license IS NOT NULL)::int,
|
||||
'has_auth', COUNT(*) FILTER (WHERE author IS NOT NULL)::int,
|
||||
'has_hp', COUNT(*) FILTER (WHERE homepage IS NOT NULL)::int
|
||||
) AS data FROM active
|
||||
),
|
||||
|
||||
-- 5b. Content length dist
|
||||
content_len AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT CASE
|
||||
WHEN raw_content IS NULL THEN 'null'
|
||||
WHEN LENGTH(raw_content)=0 THEN 'empty'
|
||||
WHEN LENGTH(raw_content)<200 THEN '<200'
|
||||
WHEN LENGTH(raw_content)<500 THEN '200-500'
|
||||
WHEN LENGTH(raw_content)<1000 THEN '500-1K'
|
||||
WHEN LENGTH(raw_content)<3000 THEN '1K-3K'
|
||||
WHEN LENGTH(raw_content)<5000 THEN '3K-5K'
|
||||
WHEN LENGTH(raw_content)<10000 THEN '5K-10K'
|
||||
WHEN LENGTH(raw_content)<50000 THEN '10K-50K'
|
||||
ELSE '50K+'
|
||||
END AS range, COUNT(*)::int AS skills
|
||||
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(LENGTH(raw_content),-1))
|
||||
) t
|
||||
),
|
||||
|
||||
-- 6. Triggers
|
||||
trigger_stats AS (
|
||||
SELECT json_build_object(
|
||||
'has_triggers', COUNT(*) FILTER (WHERE triggers IS NOT NULL)::int,
|
||||
'no_triggers', COUNT(*) FILTER (WHERE triggers IS NULL)::int,
|
||||
'file_pats', COUNT(*) FILTER (WHERE triggers->>'filePatterns' IS NOT NULL
|
||||
AND triggers->>'filePatterns'!='[]' AND triggers->>'filePatterns'!='null')::int,
|
||||
'keywords', COUNT(*) FILTER (WHERE triggers->>'keywords' IS NOT NULL
|
||||
AND triggers->>'keywords'!='[]' AND triggers->>'keywords'!='null')::int,
|
||||
'languages', COUNT(*) FILTER (WHERE triggers->>'languages' IS NOT NULL
|
||||
AND triggers->>'languages'!='[]' AND triggers->>'languages'!='null')::int,
|
||||
'has_compat', COUNT(*) FILTER (WHERE compatibility IS NOT NULL)::int,
|
||||
'platforms', COUNT(*) FILTER (WHERE compatibility->>'platforms' IS NOT NULL
|
||||
AND compatibility->>'platforms'!='[]' AND compatibility->>'platforms'!='null')::int
|
||||
) AS data FROM active
|
||||
),
|
||||
|
||||
-- 7. Freshness
|
||||
freshness AS (
|
||||
SELECT json_build_object(
|
||||
'created', (SELECT json_agg(row_to_json(t)) FROM (
|
||||
SELECT CASE
|
||||
WHEN created_at>NOW()-INTERVAL '7 days' THEN 'Last 7d'
|
||||
WHEN created_at>NOW()-INTERVAL '30 days' THEN 'Last 30d'
|
||||
WHEN created_at>NOW()-INTERVAL '90 days' THEN 'Last 90d'
|
||||
WHEN created_at>NOW()-INTERVAL '180 days' THEN 'Last 180d'
|
||||
ELSE 'Older'
|
||||
END AS range, COUNT(*)::int AS skills
|
||||
FROM active GROUP BY 1 ORDER BY MIN(NOW()-created_at)
|
||||
) t),
|
||||
'updated', (SELECT json_agg(row_to_json(t)) FROM (
|
||||
SELECT CASE
|
||||
WHEN updated_at>NOW()-INTERVAL '7 days' THEN 'Last 7d'
|
||||
WHEN updated_at>NOW()-INTERVAL '30 days' THEN 'Last 30d'
|
||||
WHEN updated_at>NOW()-INTERVAL '90 days' THEN 'Last 90d'
|
||||
WHEN updated_at>NOW()-INTERVAL '180 days' THEN 'Last 180d'
|
||||
ELSE 'Older'
|
||||
END AS range, COUNT(*)::int AS skills
|
||||
FROM active GROUP BY 1 ORDER BY MIN(NOW()-updated_at)
|
||||
) t),
|
||||
'last_dl', (SELECT json_agg(row_to_json(t)) FROM (
|
||||
SELECT CASE
|
||||
WHEN last_downloaded_at IS NULL THEN 'Never'
|
||||
WHEN last_downloaded_at>NOW()-INTERVAL '30 days' THEN 'Last 30d'
|
||||
WHEN last_downloaded_at>NOW()-INTERVAL '90 days' THEN 'Last 90d'
|
||||
ELSE 'Older'
|
||||
END AS range, COUNT(*)::int AS skills
|
||||
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(last_downloaded_at,'1970-01-01'::timestamp))
|
||||
) t)
|
||||
) AS data
|
||||
),
|
||||
|
||||
-- 8. Top owners
|
||||
top_owners_count AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT github_owner AS owner, COUNT(*)::int AS skills,
|
||||
COUNT(DISTINCT github_repo)::int AS repos,
|
||||
MAX(github_stars)::int AS max_stars,
|
||||
COALESCE(SUM(download_count),0)::int AS dl
|
||||
FROM active GROUP BY github_owner ORDER BY skills DESC LIMIT 30
|
||||
) t
|
||||
),
|
||||
top_owners_stars AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT github_owner AS owner, MAX(github_stars)::int AS max_stars,
|
||||
COUNT(*)::int AS skills, COUNT(DISTINCT github_repo)::int AS repos
|
||||
FROM active GROUP BY github_owner ORDER BY max_stars DESC LIMIT 20
|
||||
) t
|
||||
),
|
||||
owner_concentration AS (
|
||||
SELECT json_build_object(
|
||||
'top1', SUM(cnt) FILTER (WHERE rn<=1)::int,
|
||||
'top5', SUM(cnt) FILTER (WHERE rn<=5)::int,
|
||||
'top10', SUM(cnt) FILTER (WHERE rn<=10)::int,
|
||||
'top20', SUM(cnt) FILTER (WHERE rn<=20)::int,
|
||||
'top50', SUM(cnt) FILTER (WHERE rn<=50)::int,
|
||||
'total', SUM(cnt)::int,
|
||||
'owners', COUNT(*)::int
|
||||
) AS data FROM (
|
||||
SELECT github_owner, COUNT(*)::int AS cnt,
|
||||
ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn
|
||||
FROM active GROUP BY github_owner
|
||||
) x
|
||||
),
|
||||
|
||||
-- 9. Multi-skill repos
|
||||
aggregators AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT github_owner||'/'||github_repo AS repo, COUNT(*)::int AS skills,
|
||||
MAX(github_stars)::int AS stars
|
||||
FROM active GROUP BY github_owner, github_repo HAVING COUNT(*)>=20
|
||||
ORDER BY skills DESC LIMIT 30
|
||||
) t
|
||||
),
|
||||
collections AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT github_owner||'/'||github_repo AS repo, COUNT(*)::int AS skills,
|
||||
MAX(github_stars)::int AS stars
|
||||
FROM active GROUP BY github_owner, github_repo HAVING COUNT(*) BETWEEN 3 AND 19
|
||||
ORDER BY skills DESC LIMIT 30
|
||||
) t
|
||||
),
|
||||
skills_per_repo AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
WITH rc AS (SELECT COUNT(*)::int AS cnt FROM active GROUP BY github_owner, github_repo)
|
||||
SELECT CASE
|
||||
WHEN cnt=1 THEN '1' WHEN cnt=2 THEN '2'
|
||||
WHEN cnt BETWEEN 3 AND 5 THEN '3-5' WHEN cnt BETWEEN 6 AND 10 THEN '6-10'
|
||||
WHEN cnt BETWEEN 11 AND 20 THEN '11-20' WHEN cnt BETWEEN 21 AND 50 THEN '21-50'
|
||||
WHEN cnt BETWEEN 51 AND 100 THEN '51-100' ELSE '100+'
|
||||
END AS per_repo, COUNT(*)::int AS repos, SUM(cnt)::int AS total_skills
|
||||
FROM rc GROUP BY 1 ORDER BY MIN(cnt)
|
||||
) t
|
||||
),
|
||||
|
||||
-- 10. Categories
|
||||
categories_dist AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT c.name AS category, c.skill_count::int AS skills
|
||||
FROM categories c WHERE c.id NOT LIKE 'parent-%' ORDER BY c.skill_count DESC
|
||||
) t
|
||||
),
|
||||
uncategorized AS (
|
||||
SELECT COUNT(*)::int AS data FROM active s
|
||||
WHERE NOT EXISTS (SELECT 1 FROM skill_categories sc WHERE sc.skill_id=s.id)
|
||||
),
|
||||
|
||||
-- 11. Flags
|
||||
flags AS (
|
||||
SELECT json_build_object(
|
||||
'verified', COUNT(*) FILTER (WHERE is_verified)::int,
|
||||
'featured', COUNT(*) FILTER (WHERE is_featured)::int,
|
||||
'user_rated', COUNT(*) FILTER (WHERE rating_count>0)::int,
|
||||
'avg_rating', ROUND(AVG(rating) FILTER (WHERE rating IS NOT NULL),2)::numeric,
|
||||
'max_ratings', MAX(rating_count)::int
|
||||
) AS data FROM active
|
||||
),
|
||||
|
||||
-- 12. Cross-table
|
||||
cross_stats AS (
|
||||
SELECT json_build_object(
|
||||
'users', (SELECT COUNT(*)::int FROM users),
|
||||
'ratings', (SELECT COUNT(*)::int FROM ratings),
|
||||
'installs', (SELECT COUNT(*)::int FROM installations),
|
||||
'favorites', (SELECT COUNT(*)::int FROM favorites),
|
||||
'disc_repos', (SELECT COUNT(*)::int FROM discovered_repos),
|
||||
'disc_with_skills', (SELECT COUNT(*) FILTER (WHERE has_skill_md)::int FROM discovered_repos),
|
||||
'removals', (SELECT COUNT(*)::int FROM removal_requests),
|
||||
'adds', (SELECT COUNT(*)::int FROM add_requests),
|
||||
'subs', (SELECT COUNT(*) FILTER (WHERE unsubscribed_at IS NULL)::int FROM email_subscriptions)
|
||||
) AS data
|
||||
),
|
||||
install_platform AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT platform, COUNT(*)::int AS count FROM installations GROUP BY platform ORDER BY count DESC
|
||||
) t
|
||||
),
|
||||
|
||||
-- 13. Usability signals
|
||||
usability AS (
|
||||
SELECT json_build_object(
|
||||
'strong', COUNT(*) FILTER (WHERE LENGTH(description)>50 AND raw_content IS NOT NULL
|
||||
AND LENGTH(raw_content)>500 AND security_status='pass')::int,
|
||||
'file_triggers', COUNT(*) FILTER (WHERE triggers IS NOT NULL AND triggers!='{}'
|
||||
AND triggers->>'filePatterns' IS NOT NULL AND triggers->>'filePatterns'!='[]')::int,
|
||||
'downloaded', COUNT(*) FILTER (WHERE COALESCE(download_count,0)>0)::int,
|
||||
'high_q', COUNT(*) FILTER (WHERE github_stars>=10 AND LENGTH(description)>50
|
||||
AND raw_content IS NOT NULL AND security_status='pass')::int,
|
||||
'premium', COUNT(*) FILTER (WHERE github_stars>=100 AND COALESCE(download_count,0)>0
|
||||
AND security_status='pass')::int,
|
||||
'skillmd_q', COUNT(*) FILTER (WHERE source_format='skill.md' AND LENGTH(description)>50
|
||||
AND raw_content IS NOT NULL AND LENGTH(raw_content)>300 AND security_status='pass')::int
|
||||
) AS data FROM active
|
||||
),
|
||||
|
||||
-- 14. Standalone vs project-bound
|
||||
repo_types AS (
|
||||
SELECT json_build_object(
|
||||
'single', COUNT(*) FILTER (WHERE cnt=1)::int,
|
||||
'two', COUNT(*) FILTER (WHERE cnt=2)::int,
|
||||
'multi', COUNT(*) FILTER (WHERE cnt>=3)::int,
|
||||
'total', COUNT(*)::int
|
||||
) AS data FROM (
|
||||
SELECT COUNT(*)::int AS cnt FROM active GROUP BY github_owner, github_repo
|
||||
) x
|
||||
),
|
||||
name_patterns AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT CASE
|
||||
WHEN name ILIKE '%rule%' OR name ILIKE '%config%' OR name ILIKE '%setup%' THEN 'rules/config/setup'
|
||||
WHEN name ILIKE '%my-%' OR name ILIKE '%my_%' THEN 'my-prefix'
|
||||
WHEN name ILIKE '%cursor%' OR name ILIKE '%claude%' OR name ILIKE '%copilot%' THEN 'tool-specific'
|
||||
WHEN name ILIKE '%project%' OR name ILIKE '%team%' OR name ILIKE '%internal%' THEN 'project/team'
|
||||
WHEN name ILIKE '%.mdc' OR name ILIKE '%cursorrule%' THEN 'cursor-rules'
|
||||
ELSE 'generic'
|
||||
END AS pattern, COUNT(*)::int AS skills
|
||||
FROM active GROUP BY 1 ORDER BY skills DESC
|
||||
) t
|
||||
),
|
||||
|
||||
-- 15. Samples
|
||||
top_by_stars AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT id, name, github_stars AS stars, COALESCE(download_count,0) AS dl,
|
||||
security_status AS sec, source_format AS fmt, LEFT(description,80) AS description
|
||||
FROM active ORDER BY github_stars DESC LIMIT 15
|
||||
) t
|
||||
),
|
||||
top_by_dl AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT id, name, download_count AS dl, view_count AS views,
|
||||
github_stars AS stars, LEFT(description,80) AS description
|
||||
FROM active WHERE COALESCE(download_count,0)>0
|
||||
ORDER BY download_count DESC LIMIT 15
|
||||
) t
|
||||
),
|
||||
|
||||
-- 16. Discovered repos
|
||||
discovered AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT discovered_via AS source, COUNT(*)::int AS total,
|
||||
COUNT(*) FILTER (WHERE has_skill_md)::int AS with_skills,
|
||||
COUNT(*) FILTER (WHERE last_scanned IS NOT NULL)::int AS scanned
|
||||
FROM discovered_repos GROUP BY discovered_via ORDER BY total DESC
|
||||
) t
|
||||
),
|
||||
|
||||
-- 18. Cache
|
||||
cache_stats AS (
|
||||
SELECT json_build_object(
|
||||
'cached', COUNT(*) FILTER (WHERE cached_files IS NOT NULL)::int,
|
||||
'not_cached', COUNT(*) FILTER (WHERE cached_files IS NULL)::int
|
||||
) AS data FROM active
|
||||
),
|
||||
|
||||
-- 19. Branches
|
||||
branches AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT COALESCE(branch,'null') AS branch, COUNT(*)::int AS count
|
||||
FROM active GROUP BY branch ORDER BY count DESC LIMIT 10
|
||||
) t
|
||||
),
|
||||
|
||||
-- 20. Duplicates
|
||||
exact_dupes AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT name, LEFT(description,60) AS desc, COUNT(*)::int AS copies,
|
||||
STRING_AGG(DISTINCT github_owner,', ') AS owners
|
||||
FROM active WHERE description IS NOT NULL AND LENGTH(description)>10
|
||||
GROUP BY name, description HAVING COUNT(*)>1
|
||||
ORDER BY copies DESC LIMIT 15
|
||||
) t
|
||||
),
|
||||
hash_dupes AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT content_hash, COUNT(*)::int AS copies,
|
||||
MIN(name) AS sample, STRING_AGG(DISTINCT github_owner,', ') AS owners
|
||||
FROM active WHERE content_hash IS NOT NULL
|
||||
GROUP BY content_hash HAVING COUNT(*)>2
|
||||
ORDER BY copies DESC LIMIT 15
|
||||
) t
|
||||
)
|
||||
|
||||
-- Final: combine everything into one JSON
|
||||
SELECT json_build_object(
|
||||
'overall', (SELECT data FROM overall),
|
||||
'sourceFormats', (SELECT data FROM source_formats),
|
||||
'starsDist', (SELECT data FROM stars_dist),
|
||||
'dlDist', (SELECT data FROM dl_dist),
|
||||
'vwDist', (SELECT data FROM vw_dist),
|
||||
'topDL', (SELECT data FROM top_dl),
|
||||
'topVW', (SELECT data FROM top_vw),
|
||||
'security', (SELECT data FROM security),
|
||||
'contentStats', (SELECT data FROM content_stats),
|
||||
'contentLen', (SELECT data FROM content_len),
|
||||
'triggerStats', (SELECT data FROM trigger_stats),
|
||||
'freshness', (SELECT data FROM freshness),
|
||||
'topOwnersCount', (SELECT data FROM top_owners_count),
|
||||
'topOwnersStars', (SELECT data FROM top_owners_stars),
|
||||
'ownerConcentration', (SELECT data FROM owner_concentration),
|
||||
'aggregators', (SELECT data FROM aggregators),
|
||||
'collections', (SELECT data FROM collections),
|
||||
'skillsPerRepo', (SELECT data FROM skills_per_repo),
|
||||
'categories', (SELECT data FROM categories_dist),
|
||||
'uncategorized', (SELECT data FROM uncategorized),
|
||||
'flags', (SELECT data FROM flags),
|
||||
'crossStats', (SELECT data FROM cross_stats),
|
||||
'installPlatform', (SELECT data FROM install_platform),
|
||||
'usability', (SELECT data FROM usability),
|
||||
'repoTypes', (SELECT data FROM repo_types),
|
||||
'namePatterns', (SELECT data FROM name_patterns),
|
||||
'topByStars', (SELECT data FROM top_by_stars),
|
||||
'topByDL', (SELECT data FROM top_by_dl),
|
||||
'discovered', (SELECT data FROM discovered),
|
||||
'cacheStats', (SELECT data FROM cache_stats),
|
||||
'branches', (SELECT data FROM branches),
|
||||
'exactDupes', (SELECT data FROM exact_dupes),
|
||||
'hashDupes', (SELECT data FROM hash_dupes)
|
||||
) AS report;
|
||||
`;
|
||||
|
||||
// ─── Helpers ─────────────────────────────────────────────
|
||||
|
||||
function header(title) {
|
||||
console.log(`\n${'='.repeat(70)}\n ${title}\n${'='.repeat(70)}`);
|
||||
}
|
||||
function sub(title) {
|
||||
console.log(`\n -- ${title} ${'─'.repeat(Math.max(0, 58 - title.length))}`);
|
||||
}
|
||||
function tbl(rows, max = 30) {
|
||||
if (!rows || rows.length === 0) { console.log(' (no data)'); return; }
|
||||
const display = rows.slice(0, max);
|
||||
const keys = Object.keys(display[0]);
|
||||
const w = keys.map(k => Math.max(k.length, ...display.map(r => String(r[k] ?? '').length)));
|
||||
console.log(' ' + keys.map((k, i) => k.padEnd(w[i])).join(' '));
|
||||
console.log(' ' + w.map(x => '-'.repeat(x)).join(' '));
|
||||
for (const row of display)
|
||||
console.log(' ' + keys.map((k, i) => String(row[k] ?? '').padEnd(w[i])).join(' '));
|
||||
if (rows.length > max) console.log(` ... and ${rows.length - max} more`);
|
||||
}
|
||||
const n = v => Number(v ?? 0).toLocaleString('en-US');
|
||||
const p = (a, b) => (Number(b ?? 1) === 0 ? '0.0%' : (Number(a ?? 0) / Number(b) * 100).toFixed(1) + '%');
|
||||
|
||||
// ─── Run ─────────────────────────────────────────────────
|
||||
|
||||
async function run() {
|
||||
const t0 = Date.now();
|
||||
console.log('Connecting and running mega-query (this may take 30-60s)...');
|
||||
|
||||
const c = new pg.Client({ connectionString: DATABASE_URL, ssl: false,
|
||||
connectionTimeoutMillis: 15000, query_timeout: 120000,
|
||||
keepAlive: true, keepAliveInitialDelayMillis: 1000 });
|
||||
await c.connect();
|
||||
const result = await c.query(MEGA_SQL);
|
||||
await c.end().catch(() => {});
|
||||
|
||||
const r = result.rows[0].report;
|
||||
const dur = ((Date.now() - t0) / 1000).toFixed(1);
|
||||
console.log(`Query completed in ${dur}s\n`);
|
||||
|
||||
// ── Display ──
|
||||
const total = r.overall.active;
|
||||
|
||||
header('1. OVERALL STATISTICS');
|
||||
const ov = r.overall;
|
||||
console.log(` Total skills: ${n(ov.total)}`);
|
||||
console.log(` Active (not blocked): ${n(ov.active)}`);
|
||||
console.log(` Blocked: ${n(ov.blocked)}`);
|
||||
console.log(` Unique owners: ${n(ov.owners)}`);
|
||||
console.log(` Unique repos: ${n(ov.repos)}`);
|
||||
console.log(` SKILL.md format: ${n(ov.skillmd)} (${p(ov.skillmd, ov.total)})`);
|
||||
console.log(` Other formats: ${n(ov.other_fmt)} (${p(ov.other_fmt, ov.total)})`);
|
||||
console.log(` Total downloads: ${n(ov.downloads)}`);
|
||||
console.log(` Total views: ${n(ov.views)}`);
|
||||
console.log(` Rated skills: ${n(ov.rated)}`);
|
||||
console.log(` Date range: ${ov.first} -> ${ov.last}`);
|
||||
sub('Source Formats');
|
||||
tbl(r.sourceFormats);
|
||||
|
||||
header('2. GITHUB STARS');
|
||||
tbl(r.starsDist);
|
||||
|
||||
header('3. ENGAGEMENT');
|
||||
sub('Downloads'); tbl(r.dlDist);
|
||||
sub('Views'); tbl(r.vwDist);
|
||||
sub('Top 20 Downloaded'); tbl(r.topDL);
|
||||
sub('Top 20 Viewed'); tbl(r.topVW);
|
||||
|
||||
header('4. SECURITY');
|
||||
tbl(r.security);
|
||||
|
||||
header('5. CONTENT');
|
||||
const cs = r.contentStats;
|
||||
console.log(` Has content: ${n(cs.has_content)} (${p(cs.has_content, total)})`);
|
||||
console.log(` No content: ${n(cs.no_content)} (${p(cs.no_content, total)})`);
|
||||
console.log(` Avg length: ${n(cs.avg_len)} | Median: ${n(cs.median_len)} | Max: ${n(cs.max_len)}`);
|
||||
console.log(` Avg desc: ${n(cs.avg_desc)} | Good(>100): ${n(cs.good_desc)} | Short(<=20): ${n(cs.short_desc)}`);
|
||||
console.log(` Version: ${n(cs.has_ver)} | License: ${n(cs.has_lic)} | Author: ${n(cs.has_auth)} | HP: ${n(cs.has_hp)}`);
|
||||
sub('Content Length Dist');
|
||||
tbl(r.contentLen);
|
||||
|
||||
header('6. TRIGGERS');
|
||||
const tr = r.triggerStats;
|
||||
console.log(` Has triggers: ${n(tr.has_triggers)} (${p(tr.has_triggers, total)})`);
|
||||
console.log(` filePatterns: ${n(tr.file_pats)} | keywords: ${n(tr.keywords)} | languages: ${n(tr.languages)}`);
|
||||
console.log(` Compatibility: ${n(tr.has_compat)} | platforms: ${n(tr.platforms)}`);
|
||||
|
||||
header('7. FRESHNESS');
|
||||
sub('Created'); tbl(r.freshness.created);
|
||||
sub('Updated'); tbl(r.freshness.updated);
|
||||
sub('Last DL'); tbl(r.freshness.last_dl);
|
||||
|
||||
header('8. TOP OWNERS');
|
||||
sub('By Skill Count'); tbl(r.topOwnersCount);
|
||||
sub('By Stars'); tbl(r.topOwnersStars);
|
||||
sub('Concentration');
|
||||
const oc = r.ownerConcentration;
|
||||
console.log(` Top 1: ${n(oc.top1)} (${p(oc.top1, oc.total)}) | Top 5: ${n(oc.top5)} (${p(oc.top5, oc.total)}) | Top 10: ${n(oc.top10)} (${p(oc.top10, oc.total)})`);
|
||||
console.log(` Top 20: ${n(oc.top20)} (${p(oc.top20, oc.total)}) | Top 50: ${n(oc.top50)} (${p(oc.top50, oc.total)}) | Total owners: ${n(oc.owners)}`);
|
||||
|
||||
header('9. MULTI-SKILL REPOS');
|
||||
sub('Aggregators (20+)'); tbl(r.aggregators);
|
||||
sub('Collections (3-19)'); tbl(r.collections);
|
||||
sub('Skills-per-Repo'); tbl(r.skillsPerRepo);
|
||||
|
||||
header('10. CATEGORIES');
|
||||
tbl(r.categories);
|
||||
console.log(`\n Uncategorized: ${n(r.uncategorized)}`);
|
||||
|
||||
header('11. FLAGS');
|
||||
const fl = r.flags;
|
||||
console.log(` Verified: ${n(fl.verified)} | Featured: ${n(fl.featured)} | Rated: ${n(fl.user_rated)} | Avg: ${fl.avg_rating ?? 'N/A'}`);
|
||||
|
||||
header('12. CROSS-TABLE');
|
||||
const xt = r.crossStats;
|
||||
console.log(` Users: ${n(xt.users)} | Ratings: ${n(xt.ratings)} | Installs: ${n(xt.installs)} | Favs: ${n(xt.favorites)}`);
|
||||
console.log(` Discovered: ${n(xt.disc_repos)} (${n(xt.disc_with_skills)} with skills) | Subs: ${n(xt.subs)}`);
|
||||
sub('Install Platform'); tbl(r.installPlatform);
|
||||
|
||||
header('13. USABILITY SIGNALS');
|
||||
const us = r.usability;
|
||||
console.log(` Strong standalone: ${n(us.strong)} (desc>50 + content>500 + sec=pass)`);
|
||||
console.log(` File triggers: ${n(us.file_triggers)}`);
|
||||
console.log(` Ever downloaded: ${n(us.downloaded)}`);
|
||||
console.log(` High quality: ${n(us.high_q)} (stars>=10 + desc + sec)`);
|
||||
console.log(` Premium: ${n(us.premium)} (stars>=100 + dl>0 + sec)`);
|
||||
console.log(` SKILL.md quality: ${n(us.skillmd_q)}`);
|
||||
|
||||
header('14. STANDALONE vs PROJECT-BOUND');
|
||||
const sm = r.repoTypes;
|
||||
console.log(` Single: ${n(sm.single)} (${p(sm.single,sm.total)}) | Two: ${n(sm.two)} | Multi: ${n(sm.multi)} | Total repos: ${n(sm.total)}`);
|
||||
sub('Name Patterns'); tbl(r.namePatterns);
|
||||
|
||||
header('15. SAMPLES');
|
||||
sub('Top by Stars'); tbl(r.topByStars);
|
||||
sub('Top by DL'); tbl(r.topByDL);
|
||||
|
||||
header('16. DISCOVERED REPOS');
|
||||
tbl(r.discovered);
|
||||
|
||||
header('17. CACHE');
|
||||
const cf = r.cacheStats;
|
||||
console.log(` Cached: ${n(cf.cached)} (${p(cf.cached,total)}) | Not: ${n(cf.not_cached)}`);
|
||||
|
||||
header('19. BRANCHES');
|
||||
tbl(r.branches);
|
||||
|
||||
header('20. DUPLICATES');
|
||||
sub('Same Name+Desc'); tbl(r.exactDupes);
|
||||
sub('Same Hash (3+)'); tbl(r.hashDupes);
|
||||
|
||||
header(`COMPLETE (${dur}s)`);
|
||||
|
||||
// Save JSON
|
||||
r.generatedAt = new Date().toISOString();
|
||||
r.durationSeconds = parseFloat(dur);
|
||||
const reportPath = resolve(__dirname, 'explore-report.json');
|
||||
writeFileSync(reportPath, JSON.stringify(r, null, 2), 'utf-8');
|
||||
console.log(`\n Report saved: ${reportPath}`);
|
||||
}
|
||||
|
||||
run().catch(async e => {
|
||||
console.error('FAILED:', e.message || e);
|
||||
process.exit(1);
|
||||
});
|
||||
258
scripts/curation/explore.sql
Normal file
258
scripts/curation/explore.sql
Normal file
@@ -0,0 +1,258 @@
|
||||
-- Phase 1: Database Exploration - Run inside container:
|
||||
-- psql -U postgres skillhub -t -A -f /tmp/explore.sql > /tmp/report.json
|
||||
-- Or paste this SQL directly in psql
|
||||
|
||||
WITH active AS (SELECT * FROM skills WHERE is_blocked = false),
|
||||
overall AS (
|
||||
SELECT json_build_object(
|
||||
'total', (SELECT COUNT(*)::int FROM skills),
|
||||
'active', (SELECT COUNT(*)::int FROM active),
|
||||
'blocked', (SELECT COUNT(*) FILTER (WHERE is_blocked)::int FROM skills),
|
||||
'owners', (SELECT COUNT(DISTINCT github_owner)::int FROM active),
|
||||
'repos', (SELECT COUNT(DISTINCT github_owner||'/'||github_repo)::int FROM active),
|
||||
'skillmd', (SELECT COUNT(*) FILTER (WHERE source_format='skill.md')::int FROM active),
|
||||
'other_fmt', (SELECT COUNT(*) FILTER (WHERE source_format!='skill.md')::int FROM active),
|
||||
'downloads', (SELECT COALESCE(SUM(download_count),0)::bigint FROM active),
|
||||
'views', (SELECT COALESCE(SUM(view_count),0)::bigint FROM active),
|
||||
'rated', (SELECT COUNT(*) FILTER (WHERE rating_count>0)::int FROM active),
|
||||
'first', (SELECT MIN(created_at)::text FROM active),
|
||||
'last', (SELECT MAX(created_at)::text FROM active)
|
||||
) AS data
|
||||
),
|
||||
source_formats AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT COALESCE(source_format,'null') AS format, COUNT(*)::int AS count
|
||||
FROM skills GROUP BY source_format ORDER BY count DESC
|
||||
) t
|
||||
),
|
||||
stars_dist AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT CASE
|
||||
WHEN github_stars IS NULL OR github_stars=0 THEN '0'
|
||||
WHEN github_stars BETWEEN 1 AND 10 THEN '1-10'
|
||||
WHEN github_stars BETWEEN 11 AND 50 THEN '11-50'
|
||||
WHEN github_stars BETWEEN 51 AND 100 THEN '51-100'
|
||||
WHEN github_stars BETWEEN 101 AND 500 THEN '101-500'
|
||||
WHEN github_stars BETWEEN 501 AND 1000 THEN '501-1K'
|
||||
WHEN github_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
|
||||
WHEN github_stars BETWEEN 5001 AND 50000 THEN '5K-50K'
|
||||
ELSE '50K+'
|
||||
END AS stars, COUNT(*)::int AS skills,
|
||||
COUNT(DISTINCT github_owner)::int AS owners
|
||||
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(github_stars,0))
|
||||
) t
|
||||
),
|
||||
dl_dist AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT CASE
|
||||
WHEN COALESCE(download_count,0)=0 THEN '0'
|
||||
WHEN download_count BETWEEN 1 AND 10 THEN '1-10'
|
||||
WHEN download_count BETWEEN 11 AND 100 THEN '11-100'
|
||||
ELSE '100+'
|
||||
END AS range, COUNT(*)::int AS skills
|
||||
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(download_count,0))
|
||||
) t
|
||||
),
|
||||
top_dl AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT id, name, download_count AS dl, github_stars AS stars, github_owner AS owner
|
||||
FROM active WHERE COALESCE(download_count,0)>0 ORDER BY download_count DESC LIMIT 20
|
||||
) t
|
||||
),
|
||||
top_vw AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT id, name, view_count AS views, github_stars AS stars, github_owner AS owner
|
||||
FROM active WHERE COALESCE(view_count,0)>0 ORDER BY view_count DESC LIMIT 20
|
||||
) t
|
||||
),
|
||||
security AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT COALESCE(security_status,'null') AS status, COUNT(*)::int AS count
|
||||
FROM active GROUP BY security_status ORDER BY count DESC
|
||||
) t
|
||||
),
|
||||
content_stats AS (
|
||||
SELECT json_build_object(
|
||||
'has_content', COUNT(*) FILTER (WHERE raw_content IS NOT NULL)::int,
|
||||
'no_content', COUNT(*) FILTER (WHERE raw_content IS NULL)::int,
|
||||
'avg_len', ROUND(AVG(LENGTH(raw_content)) FILTER (WHERE raw_content IS NOT NULL))::int,
|
||||
'max_len', MAX(LENGTH(raw_content))::int,
|
||||
'median_len', PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY LENGTH(raw_content)) FILTER (WHERE raw_content IS NOT NULL)::int,
|
||||
'avg_desc', ROUND(AVG(LENGTH(description)))::int,
|
||||
'good_desc', COUNT(*) FILTER (WHERE LENGTH(description)>100)::int,
|
||||
'short_desc', COUNT(*) FILTER (WHERE LENGTH(description)<=20)::int,
|
||||
'has_ver', COUNT(*) FILTER (WHERE version IS NOT NULL)::int,
|
||||
'has_lic', COUNT(*) FILTER (WHERE license IS NOT NULL)::int,
|
||||
'has_auth', COUNT(*) FILTER (WHERE author IS NOT NULL)::int
|
||||
) AS data FROM active
|
||||
),
|
||||
content_len AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT CASE
|
||||
WHEN raw_content IS NULL THEN 'null'
|
||||
WHEN LENGTH(raw_content)<200 THEN '<200'
|
||||
WHEN LENGTH(raw_content)<1000 THEN '200-1K'
|
||||
WHEN LENGTH(raw_content)<5000 THEN '1K-5K'
|
||||
WHEN LENGTH(raw_content)<10000 THEN '5K-10K'
|
||||
WHEN LENGTH(raw_content)<50000 THEN '10K-50K'
|
||||
ELSE '50K+'
|
||||
END AS range, COUNT(*)::int AS skills
|
||||
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(LENGTH(raw_content),-1))
|
||||
) t
|
||||
),
|
||||
trigger_stats AS (
|
||||
SELECT json_build_object(
|
||||
'has_triggers', COUNT(*) FILTER (WHERE triggers IS NOT NULL)::int,
|
||||
'no_triggers', COUNT(*) FILTER (WHERE triggers IS NULL)::int,
|
||||
'file_pats', COUNT(*) FILTER (WHERE triggers->>'filePatterns' IS NOT NULL AND triggers->>'filePatterns'!='[]')::int,
|
||||
'keywords', COUNT(*) FILTER (WHERE triggers->>'keywords' IS NOT NULL AND triggers->>'keywords'!='[]')::int,
|
||||
'platforms', COUNT(*) FILTER (WHERE compatibility->>'platforms' IS NOT NULL AND compatibility->>'platforms'!='[]')::int
|
||||
) AS data FROM active
|
||||
),
|
||||
freshness AS (
|
||||
SELECT json_build_object(
|
||||
'created', (SELECT json_agg(row_to_json(t)) FROM (
|
||||
SELECT CASE WHEN created_at>NOW()-INTERVAL '7d' THEN '7d' WHEN created_at>NOW()-INTERVAL '30d' THEN '30d'
|
||||
WHEN created_at>NOW()-INTERVAL '90d' THEN '90d' ELSE 'older' END AS range, COUNT(*)::int AS skills
|
||||
FROM active GROUP BY 1 ORDER BY MIN(NOW()-created_at)) t),
|
||||
'updated', (SELECT json_agg(row_to_json(t)) FROM (
|
||||
SELECT CASE WHEN updated_at>NOW()-INTERVAL '7d' THEN '7d' WHEN updated_at>NOW()-INTERVAL '30d' THEN '30d'
|
||||
WHEN updated_at>NOW()-INTERVAL '90d' THEN '90d' ELSE 'older' END AS range, COUNT(*)::int AS skills
|
||||
FROM active GROUP BY 1 ORDER BY MIN(NOW()-updated_at)) t)
|
||||
) AS data
|
||||
),
|
||||
top_owners AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT github_owner AS owner, COUNT(*)::int AS skills, COUNT(DISTINCT github_repo)::int AS repos,
|
||||
MAX(github_stars)::int AS max_stars, COALESCE(SUM(download_count),0)::int AS dl
|
||||
FROM active GROUP BY github_owner ORDER BY skills DESC LIMIT 30
|
||||
) t
|
||||
),
|
||||
owner_concentration AS (
|
||||
SELECT json_build_object(
|
||||
'top1', SUM(cnt) FILTER (WHERE rn<=1)::int, 'top5', SUM(cnt) FILTER (WHERE rn<=5)::int,
|
||||
'top10', SUM(cnt) FILTER (WHERE rn<=10)::int, 'top20', SUM(cnt) FILTER (WHERE rn<=20)::int,
|
||||
'total', SUM(cnt)::int, 'owners', COUNT(*)::int
|
||||
) AS data FROM (
|
||||
SELECT github_owner, COUNT(*)::int AS cnt, ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn
|
||||
FROM active GROUP BY github_owner
|
||||
) x
|
||||
),
|
||||
aggregators AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT github_owner||'/'||github_repo AS repo, COUNT(*)::int AS skills, MAX(github_stars)::int AS stars
|
||||
FROM active GROUP BY github_owner, github_repo HAVING COUNT(*)>=20 ORDER BY skills DESC LIMIT 20
|
||||
) t
|
||||
),
|
||||
collections AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT github_owner||'/'||github_repo AS repo, COUNT(*)::int AS skills, MAX(github_stars)::int AS stars
|
||||
FROM active GROUP BY github_owner, github_repo HAVING COUNT(*) BETWEEN 3 AND 19 ORDER BY skills DESC LIMIT 20
|
||||
) t
|
||||
),
|
||||
skills_per_repo AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
WITH rc AS (SELECT COUNT(*)::int AS cnt FROM active GROUP BY github_owner, github_repo)
|
||||
SELECT CASE WHEN cnt=1 THEN '1' WHEN cnt=2 THEN '2' WHEN cnt BETWEEN 3 AND 10 THEN '3-10'
|
||||
WHEN cnt BETWEEN 11 AND 50 THEN '11-50' ELSE '50+' END AS per_repo,
|
||||
COUNT(*)::int AS repos, SUM(cnt)::int AS total_skills
|
||||
FROM rc GROUP BY 1 ORDER BY MIN(cnt)
|
||||
) t
|
||||
),
|
||||
categories_dist AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT c.name AS category, c.skill_count::int AS skills
|
||||
FROM categories c WHERE c.id NOT LIKE 'parent-%' ORDER BY c.skill_count DESC
|
||||
) t
|
||||
),
|
||||
flags AS (
|
||||
SELECT json_build_object(
|
||||
'verified', COUNT(*) FILTER (WHERE is_verified)::int,
|
||||
'featured', COUNT(*) FILTER (WHERE is_featured)::int,
|
||||
'user_rated', COUNT(*) FILTER (WHERE rating_count>0)::int
|
||||
) AS data FROM active
|
||||
),
|
||||
cross_stats AS (
|
||||
SELECT json_build_object(
|
||||
'users', (SELECT COUNT(*)::int FROM users),
|
||||
'ratings', (SELECT COUNT(*)::int FROM ratings),
|
||||
'installs', (SELECT COUNT(*)::int FROM installations),
|
||||
'favorites', (SELECT COUNT(*)::int FROM favorites),
|
||||
'disc_repos', (SELECT COUNT(*)::int FROM discovered_repos),
|
||||
'subs', (SELECT COUNT(*) FILTER (WHERE unsubscribed_at IS NULL)::int FROM email_subscriptions)
|
||||
) AS data
|
||||
),
|
||||
usability AS (
|
||||
SELECT json_build_object(
|
||||
'strong', COUNT(*) FILTER (WHERE LENGTH(description)>50 AND raw_content IS NOT NULL AND LENGTH(raw_content)>500 AND security_status='pass')::int,
|
||||
'downloaded', COUNT(*) FILTER (WHERE COALESCE(download_count,0)>0)::int,
|
||||
'high_q', COUNT(*) FILTER (WHERE github_stars>=10 AND LENGTH(description)>50 AND raw_content IS NOT NULL AND security_status='pass')::int,
|
||||
'premium', COUNT(*) FILTER (WHERE github_stars>=100 AND COALESCE(download_count,0)>0 AND security_status='pass')::int,
|
||||
'skillmd_q', COUNT(*) FILTER (WHERE source_format='skill.md' AND LENGTH(description)>50 AND raw_content IS NOT NULL AND LENGTH(raw_content)>300 AND security_status='pass')::int
|
||||
) AS data FROM active
|
||||
),
|
||||
repo_types AS (
|
||||
SELECT json_build_object(
|
||||
'single', COUNT(*) FILTER (WHERE cnt=1)::int, 'two', COUNT(*) FILTER (WHERE cnt=2)::int,
|
||||
'multi', COUNT(*) FILTER (WHERE cnt>=3)::int, 'total', COUNT(*)::int
|
||||
) AS data FROM (SELECT COUNT(*)::int AS cnt FROM active GROUP BY github_owner, github_repo) x
|
||||
),
|
||||
name_patterns AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT CASE
|
||||
WHEN name ILIKE '%rule%' OR name ILIKE '%config%' OR name ILIKE '%setup%' THEN 'rules/config'
|
||||
WHEN name ILIKE '%cursor%' OR name ILIKE '%claude%' OR name ILIKE '%copilot%' THEN 'tool-specific'
|
||||
WHEN name ILIKE '%project%' OR name ILIKE '%team%' THEN 'project/team'
|
||||
ELSE 'generic'
|
||||
END AS pattern, COUNT(*)::int AS skills
|
||||
FROM active GROUP BY 1 ORDER BY skills DESC
|
||||
) t
|
||||
),
|
||||
top_by_stars AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT id, name, github_stars AS stars, COALESCE(download_count,0) AS dl,
|
||||
security_status AS sec, source_format AS fmt, LEFT(description,80) AS description
|
||||
FROM active ORDER BY github_stars DESC LIMIT 15
|
||||
) t
|
||||
),
|
||||
discovered AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT discovered_via AS source, COUNT(*)::int AS total,
|
||||
COUNT(*) FILTER (WHERE has_skill_md)::int AS with_skills
|
||||
FROM discovered_repos GROUP BY discovered_via ORDER BY total DESC
|
||||
) t
|
||||
),
|
||||
hash_dupes AS (
|
||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
||||
SELECT content_hash, COUNT(*)::int AS copies, MIN(name) AS sample
|
||||
FROM active WHERE content_hash IS NOT NULL
|
||||
GROUP BY content_hash HAVING COUNT(*)>2 ORDER BY copies DESC LIMIT 15
|
||||
) t
|
||||
)
|
||||
SELECT json_build_object(
|
||||
'overall', (SELECT data FROM overall),
|
||||
'sourceFormats', (SELECT data FROM source_formats),
|
||||
'starsDist', (SELECT data FROM stars_dist),
|
||||
'dlDist', (SELECT data FROM dl_dist),
|
||||
'topDL', (SELECT data FROM top_dl),
|
||||
'topVW', (SELECT data FROM top_vw),
|
||||
'security', (SELECT data FROM security),
|
||||
'contentStats', (SELECT data FROM content_stats),
|
||||
'contentLen', (SELECT data FROM content_len),
|
||||
'triggerStats', (SELECT data FROM trigger_stats),
|
||||
'freshness', (SELECT data FROM freshness),
|
||||
'topOwners', (SELECT data FROM top_owners),
|
||||
'ownerConcentration', (SELECT data FROM owner_concentration),
|
||||
'aggregators', (SELECT data FROM aggregators),
|
||||
'collections', (SELECT data FROM collections),
|
||||
'skillsPerRepo', (SELECT data FROM skills_per_repo),
|
||||
'categories', (SELECT data FROM categories_dist),
|
||||
'flags', (SELECT data FROM flags),
|
||||
'crossStats', (SELECT data FROM cross_stats),
|
||||
'usability', (SELECT data FROM usability),
|
||||
'repoTypes', (SELECT data FROM repo_types),
|
||||
'namePatterns', (SELECT data FROM name_patterns),
|
||||
'topByStars', (SELECT data FROM top_by_stars),
|
||||
'discovered', (SELECT data FROM discovered),
|
||||
'hashDupes', (SELECT data FROM hash_dupes)
|
||||
) AS report;
|
||||
894
scripts/curation/explore.ts
Normal file
894
scripts/curation/explore.ts
Normal file
@@ -0,0 +1,894 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* Phase 1: Database Exploration for Curation
|
||||
*
|
||||
* Comprehensive analysis of all indexed skills to understand:
|
||||
* - Overall statistics and distribution
|
||||
* - Quality signals and content analysis
|
||||
* - Owner/repo concentration
|
||||
* - Freshness and activity
|
||||
* - Usability signals for curation decisions
|
||||
*
|
||||
* Usage:
|
||||
* cd services/indexer && npx tsx ../../scripts/curation/explore.ts
|
||||
*
|
||||
* Output: prints report to console + saves JSON to scripts/curation/explore-report.json
|
||||
*/
|
||||
|
||||
import { createDb, closeDb, sql } from '@skillhub/db';
|
||||
import { writeFileSync } from 'fs';
|
||||
import { resolve, dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
// ─── Connection ────────────────────────────────────────────────────────────────
|
||||
|
||||
const db = createDb();
|
||||
|
||||
// ─── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
function header(title: string) {
|
||||
const line = '='.repeat(70);
|
||||
console.log(`\n${line}`);
|
||||
console.log(` ${title}`);
|
||||
console.log(line);
|
||||
}
|
||||
|
||||
function subHeader(title: string) {
|
||||
console.log(`\n -- ${title} ${'─'.repeat(Math.max(0, 60 - title.length))}`);
|
||||
}
|
||||
|
||||
function table(rows: Record<string, unknown>[], maxRows = 30) {
|
||||
if (rows.length === 0) {
|
||||
console.log(' (no data)');
|
||||
return;
|
||||
}
|
||||
const display = rows.slice(0, maxRows);
|
||||
const keys = Object.keys(display[0]);
|
||||
const widths = keys.map(k =>
|
||||
Math.max(k.length, ...display.map(r => String(r[k] ?? '').length))
|
||||
);
|
||||
|
||||
// Header
|
||||
console.log(' ' + keys.map((k, i) => k.padEnd(widths[i])).join(' '));
|
||||
console.log(' ' + widths.map(w => '-'.repeat(w)).join(' '));
|
||||
|
||||
// Rows
|
||||
for (const row of display) {
|
||||
console.log(' ' + keys.map((k, i) => String(row[k] ?? '').padEnd(widths[i])).join(' '));
|
||||
}
|
||||
if (rows.length > maxRows) {
|
||||
console.log(` ... and ${rows.length - maxRows} more rows`);
|
||||
}
|
||||
}
|
||||
|
||||
function num(n: unknown): string {
|
||||
return Number(n ?? 0).toLocaleString('en-US');
|
||||
}
|
||||
|
||||
function pct(part: unknown, total: unknown): string {
|
||||
const p = Number(part ?? 0);
|
||||
const t = Number(total ?? 1);
|
||||
if (t === 0) return '0.0%';
|
||||
return (p / t * 100).toFixed(1) + '%';
|
||||
}
|
||||
|
||||
// Helper: execute raw SQL and return rows
|
||||
async function query<T = Record<string, unknown>>(strings: TemplateStringsArray, ...values: unknown[]): Promise<T[]> {
|
||||
const result = await db.execute(sql.raw(String.raw(strings, ...values)));
|
||||
return result.rows as T[];
|
||||
}
|
||||
|
||||
// ─── Report accumulator ────────────────────────────────────────────────────────
|
||||
|
||||
const report: Record<string, unknown> = {};
|
||||
|
||||
// ─── Queries ───────────────────────────────────────────────────────────────────
|
||||
|
||||
async function runExploration() {
|
||||
const startTime = Date.now();
|
||||
|
||||
// ======================================================================
|
||||
// 1. OVERALL STATISTICS
|
||||
// ======================================================================
|
||||
header('1. OVERALL STATISTICS');
|
||||
|
||||
const overallRows = await db.execute(sql`
|
||||
SELECT
|
||||
COUNT(*)::int AS total_skills,
|
||||
COUNT(*) FILTER (WHERE is_blocked = false)::int AS active_skills,
|
||||
COUNT(*) FILTER (WHERE is_blocked = true)::int AS blocked_skills,
|
||||
COUNT(DISTINCT github_owner)::int AS unique_owners,
|
||||
COUNT(DISTINCT github_owner || '/' || github_repo)::int AS unique_repos,
|
||||
COUNT(*) FILTER (WHERE source_format = 'skill.md')::int AS skill_md_count,
|
||||
COUNT(*) FILTER (WHERE source_format != 'skill.md')::int AS other_format_count,
|
||||
COALESCE(SUM(download_count), 0)::bigint AS total_downloads,
|
||||
COALESCE(SUM(view_count), 0)::bigint AS total_views,
|
||||
COUNT(*) FILTER (WHERE rating_count > 0)::int AS rated_skills,
|
||||
COALESCE(SUM(rating_count), 0)::int AS total_ratings,
|
||||
MIN(created_at)::text AS earliest_skill,
|
||||
MAX(created_at)::text AS latest_skill
|
||||
FROM skills
|
||||
`);
|
||||
const overall = overallRows.rows[0] as Record<string, unknown>;
|
||||
report.overall = overall;
|
||||
|
||||
console.log(` Total skills: ${num(overall.total_skills)}`);
|
||||
console.log(` Active (not blocked): ${num(overall.active_skills)}`);
|
||||
console.log(` Blocked: ${num(overall.blocked_skills)}`);
|
||||
console.log(` Unique owners: ${num(overall.unique_owners)}`);
|
||||
console.log(` Unique repos: ${num(overall.unique_repos)}`);
|
||||
console.log(` SKILL.md format: ${num(overall.skill_md_count)} (${pct(overall.skill_md_count, overall.total_skills)})`);
|
||||
console.log(` Other formats: ${num(overall.other_format_count)} (${pct(overall.other_format_count, overall.total_skills)})`);
|
||||
console.log(` Total downloads: ${num(overall.total_downloads)}`);
|
||||
console.log(` Total views: ${num(overall.total_views)}`);
|
||||
console.log(` Rated skills: ${num(overall.rated_skills)}`);
|
||||
console.log(` Total ratings: ${num(overall.total_ratings)}`);
|
||||
console.log(` Date range: ${overall.earliest_skill} -> ${overall.latest_skill}`);
|
||||
|
||||
const total = Number(overall.active_skills);
|
||||
|
||||
// Source format distribution
|
||||
subHeader('Source Format Distribution');
|
||||
const sfRows = await db.execute(sql`
|
||||
SELECT
|
||||
COALESCE(source_format, 'null') AS format,
|
||||
COUNT(*)::int AS count,
|
||||
COUNT(*) FILTER (WHERE is_blocked = false)::int AS active
|
||||
FROM skills
|
||||
GROUP BY source_format
|
||||
ORDER BY count DESC
|
||||
`);
|
||||
report.sourceFormats = sfRows.rows;
|
||||
table(sfRows.rows as Record<string, unknown>[]);
|
||||
|
||||
// ======================================================================
|
||||
// 2. STARS DISTRIBUTION
|
||||
// ======================================================================
|
||||
header('2. GITHUB STARS DISTRIBUTION');
|
||||
|
||||
const starsRows = await db.execute(sql`
|
||||
SELECT
|
||||
CASE
|
||||
WHEN github_stars IS NULL OR github_stars = 0 THEN '0 stars'
|
||||
WHEN github_stars BETWEEN 1 AND 5 THEN '1-5'
|
||||
WHEN github_stars BETWEEN 6 AND 10 THEN '6-10'
|
||||
WHEN github_stars BETWEEN 11 AND 50 THEN '11-50'
|
||||
WHEN github_stars BETWEEN 51 AND 100 THEN '51-100'
|
||||
WHEN github_stars BETWEEN 101 AND 500 THEN '101-500'
|
||||
WHEN github_stars BETWEEN 501 AND 1000 THEN '501-1K'
|
||||
WHEN github_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
|
||||
WHEN github_stars BETWEEN 5001 AND 10000 THEN '5K-10K'
|
||||
WHEN github_stars BETWEEN 10001 AND 50000 THEN '10K-50K'
|
||||
ELSE '50K+'
|
||||
END AS star_range,
|
||||
COUNT(*)::int AS skills,
|
||||
COUNT(DISTINCT github_owner)::int AS owners,
|
||||
COUNT(DISTINCT github_owner || '/' || github_repo)::int AS repos
|
||||
FROM skills
|
||||
WHERE is_blocked = false
|
||||
GROUP BY
|
||||
CASE
|
||||
WHEN github_stars IS NULL OR github_stars = 0 THEN 0
|
||||
WHEN github_stars BETWEEN 1 AND 5 THEN 1
|
||||
WHEN github_stars BETWEEN 6 AND 10 THEN 2
|
||||
WHEN github_stars BETWEEN 11 AND 50 THEN 3
|
||||
WHEN github_stars BETWEEN 51 AND 100 THEN 4
|
||||
WHEN github_stars BETWEEN 101 AND 500 THEN 5
|
||||
WHEN github_stars BETWEEN 501 AND 1000 THEN 6
|
||||
WHEN github_stars BETWEEN 1001 AND 5000 THEN 7
|
||||
WHEN github_stars BETWEEN 5001 AND 10000 THEN 8
|
||||
WHEN github_stars BETWEEN 10001 AND 50000 THEN 9
|
||||
ELSE 10
|
||||
END,
|
||||
CASE
|
||||
WHEN github_stars IS NULL OR github_stars = 0 THEN '0 stars'
|
||||
WHEN github_stars BETWEEN 1 AND 5 THEN '1-5'
|
||||
WHEN github_stars BETWEEN 6 AND 10 THEN '6-10'
|
||||
WHEN github_stars BETWEEN 11 AND 50 THEN '11-50'
|
||||
WHEN github_stars BETWEEN 51 AND 100 THEN '51-100'
|
||||
WHEN github_stars BETWEEN 101 AND 500 THEN '101-500'
|
||||
WHEN github_stars BETWEEN 501 AND 1000 THEN '501-1K'
|
||||
WHEN github_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
|
||||
WHEN github_stars BETWEEN 5001 AND 10000 THEN '5K-10K'
|
||||
WHEN github_stars BETWEEN 10001 AND 50000 THEN '10K-50K'
|
||||
ELSE '50K+'
|
||||
END
|
||||
ORDER BY MIN(COALESCE(github_stars, 0))
|
||||
`);
|
||||
report.starsDist = starsRows.rows;
|
||||
table(starsRows.rows as Record<string, unknown>[]);
|
||||
|
||||
// ======================================================================
|
||||
// 3. ENGAGEMENT DISTRIBUTION
|
||||
// ======================================================================
|
||||
header('3. ENGAGEMENT (Downloads & Views)');
|
||||
|
||||
subHeader('Download Distribution');
|
||||
const dlRows = await db.execute(sql`
|
||||
SELECT
|
||||
CASE
|
||||
WHEN COALESCE(download_count, 0) = 0 THEN '0'
|
||||
WHEN download_count BETWEEN 1 AND 5 THEN '1-5'
|
||||
WHEN download_count BETWEEN 6 AND 10 THEN '6-10'
|
||||
WHEN download_count BETWEEN 11 AND 50 THEN '11-50'
|
||||
WHEN download_count BETWEEN 51 AND 100 THEN '51-100'
|
||||
WHEN download_count BETWEEN 101 AND 500 THEN '101-500'
|
||||
ELSE '500+'
|
||||
END AS range,
|
||||
COUNT(*)::int AS skills
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY 1 ORDER BY MIN(COALESCE(download_count, 0))
|
||||
`);
|
||||
report.downloadDist = dlRows.rows;
|
||||
table(dlRows.rows as Record<string, unknown>[]);
|
||||
|
||||
subHeader('View Distribution');
|
||||
const vwRows = await db.execute(sql`
|
||||
SELECT
|
||||
CASE
|
||||
WHEN COALESCE(view_count, 0) = 0 THEN '0'
|
||||
WHEN view_count BETWEEN 1 AND 10 THEN '1-10'
|
||||
WHEN view_count BETWEEN 11 AND 50 THEN '11-50'
|
||||
WHEN view_count BETWEEN 51 AND 100 THEN '51-100'
|
||||
WHEN view_count BETWEEN 101 AND 500 THEN '101-500'
|
||||
WHEN view_count BETWEEN 501 AND 1000 THEN '501-1K'
|
||||
ELSE '1K+'
|
||||
END AS range,
|
||||
COUNT(*)::int AS skills
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY 1 ORDER BY MIN(COALESCE(view_count, 0))
|
||||
`);
|
||||
report.viewDist = vwRows.rows;
|
||||
table(vwRows.rows as Record<string, unknown>[]);
|
||||
|
||||
subHeader('Top 20 Downloaded Skills');
|
||||
const topDL = await db.execute(sql`
|
||||
SELECT id, name, download_count AS downloads, view_count AS views,
|
||||
github_stars AS stars, github_owner AS owner
|
||||
FROM skills
|
||||
WHERE is_blocked = false AND COALESCE(download_count, 0) > 0
|
||||
ORDER BY download_count DESC LIMIT 20
|
||||
`);
|
||||
report.topDownloads = topDL.rows;
|
||||
table(topDL.rows as Record<string, unknown>[]);
|
||||
|
||||
subHeader('Top 20 Viewed Skills');
|
||||
const topVW = await db.execute(sql`
|
||||
SELECT id, name, view_count AS views, download_count AS downloads,
|
||||
github_stars AS stars, github_owner AS owner
|
||||
FROM skills
|
||||
WHERE is_blocked = false AND COALESCE(view_count, 0) > 0
|
||||
ORDER BY view_count DESC LIMIT 20
|
||||
`);
|
||||
report.topViews = topVW.rows;
|
||||
table(topVW.rows as Record<string, unknown>[]);
|
||||
|
||||
// ======================================================================
|
||||
// 4. SECURITY STATUS
|
||||
// ======================================================================
|
||||
header('4. SECURITY STATUS');
|
||||
|
||||
const secRows = await db.execute(sql`
|
||||
SELECT
|
||||
COALESCE(security_status, 'null') AS status,
|
||||
COUNT(*)::int AS count,
|
||||
ROUND(COUNT(*)::numeric / NULLIF(${total}, 0) * 100, 1) AS pct
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY security_status ORDER BY count DESC
|
||||
`);
|
||||
report.securityDist = secRows.rows;
|
||||
table(secRows.rows as Record<string, unknown>[]);
|
||||
|
||||
// ======================================================================
|
||||
// 5. CONTENT ANALYSIS
|
||||
// ======================================================================
|
||||
header('5. CONTENT ANALYSIS');
|
||||
|
||||
const csRows = await db.execute(sql`
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE raw_content IS NOT NULL)::int AS has_content,
|
||||
COUNT(*) FILTER (WHERE raw_content IS NULL)::int AS no_content,
|
||||
ROUND(AVG(LENGTH(raw_content)) FILTER (WHERE raw_content IS NOT NULL))::int AS avg_content_len,
|
||||
MAX(LENGTH(raw_content))::int AS max_content_len,
|
||||
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY LENGTH(raw_content))
|
||||
FILTER (WHERE raw_content IS NOT NULL)::int AS median_content_len,
|
||||
ROUND(AVG(LENGTH(description)))::int AS avg_desc_len,
|
||||
COUNT(*) FILTER (WHERE LENGTH(description) > 100)::int AS good_desc,
|
||||
COUNT(*) FILTER (WHERE LENGTH(description) <= 20)::int AS short_desc,
|
||||
COUNT(*) FILTER (WHERE version IS NOT NULL)::int AS has_version,
|
||||
COUNT(*) FILTER (WHERE license IS NOT NULL)::int AS has_license,
|
||||
COUNT(*) FILTER (WHERE author IS NOT NULL)::int AS has_author,
|
||||
COUNT(*) FILTER (WHERE homepage IS NOT NULL)::int AS has_homepage
|
||||
FROM skills WHERE is_blocked = false
|
||||
`);
|
||||
const cs = csRows.rows[0] as Record<string, unknown>;
|
||||
report.contentStats = cs;
|
||||
|
||||
console.log(` Has raw_content: ${num(cs.has_content)} (${pct(cs.has_content, total)})`);
|
||||
console.log(` No raw_content: ${num(cs.no_content)} (${pct(cs.no_content, total)})`);
|
||||
console.log(` Avg content length: ${num(cs.avg_content_len)} chars`);
|
||||
console.log(` Median content len: ${num(cs.median_content_len)} chars`);
|
||||
console.log(` Max content length: ${num(cs.max_content_len)} chars`);
|
||||
console.log(` Avg desc length: ${num(cs.avg_desc_len)} chars`);
|
||||
console.log(` Good desc (>100ch): ${num(cs.good_desc)} (${pct(cs.good_desc, total)})`);
|
||||
console.log(` Short desc (<=20): ${num(cs.short_desc)} (${pct(cs.short_desc, total)})`);
|
||||
console.log(` Has version: ${num(cs.has_version)} (${pct(cs.has_version, total)})`);
|
||||
console.log(` Has license: ${num(cs.has_license)} (${pct(cs.has_license, total)})`);
|
||||
console.log(` Has author: ${num(cs.has_author)} (${pct(cs.has_author, total)})`);
|
||||
console.log(` Has homepage: ${num(cs.has_homepage)} (${pct(cs.has_homepage, total)})`);
|
||||
|
||||
subHeader('Content Length Distribution');
|
||||
const clRows = await db.execute(sql`
|
||||
SELECT
|
||||
CASE
|
||||
WHEN raw_content IS NULL THEN 'null'
|
||||
WHEN LENGTH(raw_content) = 0 THEN 'empty'
|
||||
WHEN LENGTH(raw_content) < 200 THEN '<200'
|
||||
WHEN LENGTH(raw_content) < 500 THEN '200-500'
|
||||
WHEN LENGTH(raw_content) < 1000 THEN '500-1K'
|
||||
WHEN LENGTH(raw_content) < 3000 THEN '1K-3K'
|
||||
WHEN LENGTH(raw_content) < 5000 THEN '3K-5K'
|
||||
WHEN LENGTH(raw_content) < 10000 THEN '5K-10K'
|
||||
WHEN LENGTH(raw_content) < 50000 THEN '10K-50K'
|
||||
ELSE '50K+'
|
||||
END AS range,
|
||||
COUNT(*)::int AS skills
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY 1 ORDER BY MIN(COALESCE(LENGTH(raw_content), -1))
|
||||
`);
|
||||
report.contentLenDist = clRows.rows;
|
||||
table(clRows.rows as Record<string, unknown>[]);
|
||||
|
||||
// ======================================================================
|
||||
// 6. TRIGGERS & COMPATIBILITY
|
||||
// ======================================================================
|
||||
header('6. TRIGGERS & COMPATIBILITY');
|
||||
|
||||
const trRows = await db.execute(sql`
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE triggers IS NOT NULL)::int AS has_triggers,
|
||||
COUNT(*) FILTER (WHERE triggers IS NULL)::int AS no_triggers,
|
||||
COUNT(*) FILTER (WHERE triggers->>'filePatterns' IS NOT NULL
|
||||
AND triggers->>'filePatterns' != '[]'
|
||||
AND triggers->>'filePatterns' != 'null')::int AS has_file_patterns,
|
||||
COUNT(*) FILTER (WHERE triggers->>'keywords' IS NOT NULL
|
||||
AND triggers->>'keywords' != '[]'
|
||||
AND triggers->>'keywords' != 'null')::int AS has_keywords,
|
||||
COUNT(*) FILTER (WHERE triggers->>'languages' IS NOT NULL
|
||||
AND triggers->>'languages' != '[]'
|
||||
AND triggers->>'languages' != 'null')::int AS has_languages,
|
||||
COUNT(*) FILTER (WHERE compatibility IS NOT NULL)::int AS has_compat,
|
||||
COUNT(*) FILTER (WHERE compatibility->>'platforms' IS NOT NULL
|
||||
AND compatibility->>'platforms' != '[]'
|
||||
AND compatibility->>'platforms' != 'null')::int AS has_platforms,
|
||||
COUNT(*) FILTER (WHERE compatibility->>'requires' IS NOT NULL
|
||||
AND compatibility->>'requires' != '[]'
|
||||
AND compatibility->>'requires' != 'null')::int AS has_requires
|
||||
FROM skills WHERE is_blocked = false
|
||||
`);
|
||||
const tr = trRows.rows[0] as Record<string, unknown>;
|
||||
report.triggerStats = tr;
|
||||
|
||||
console.log(` Has triggers: ${num(tr.has_triggers)} (${pct(tr.has_triggers, total)})`);
|
||||
console.log(` No triggers: ${num(tr.no_triggers)} (${pct(tr.no_triggers, total)})`);
|
||||
console.log(` - filePatterns: ${num(tr.has_file_patterns)}`);
|
||||
console.log(` - keywords: ${num(tr.has_keywords)}`);
|
||||
console.log(` - languages: ${num(tr.has_languages)}`);
|
||||
console.log(` Has compatibility: ${num(tr.has_compat)}`);
|
||||
console.log(` - platforms: ${num(tr.has_platforms)}`);
|
||||
console.log(` - requires: ${num(tr.has_requires)}`);
|
||||
|
||||
// ======================================================================
|
||||
// 7. FRESHNESS
|
||||
// ======================================================================
|
||||
header('7. FRESHNESS & ACTIVITY');
|
||||
|
||||
subHeader('Created At Distribution');
|
||||
const crRows = await db.execute(sql`
|
||||
SELECT
|
||||
CASE
|
||||
WHEN created_at > NOW() - INTERVAL '7 days' THEN 'Last 7d'
|
||||
WHEN created_at > NOW() - INTERVAL '30 days' THEN 'Last 30d'
|
||||
WHEN created_at > NOW() - INTERVAL '90 days' THEN 'Last 90d'
|
||||
WHEN created_at > NOW() - INTERVAL '180 days' THEN 'Last 180d'
|
||||
WHEN created_at > NOW() - INTERVAL '365 days' THEN 'Last 365d'
|
||||
ELSE 'Older'
|
||||
END AS range,
|
||||
COUNT(*)::int AS skills
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY 1 ORDER BY MIN(NOW() - created_at)
|
||||
`);
|
||||
report.createdDist = crRows.rows;
|
||||
table(crRows.rows as Record<string, unknown>[]);
|
||||
|
||||
subHeader('Updated At Distribution');
|
||||
const upRows = await db.execute(sql`
|
||||
SELECT
|
||||
CASE
|
||||
WHEN updated_at > NOW() - INTERVAL '7 days' THEN 'Last 7d'
|
||||
WHEN updated_at > NOW() - INTERVAL '30 days' THEN 'Last 30d'
|
||||
WHEN updated_at > NOW() - INTERVAL '90 days' THEN 'Last 90d'
|
||||
WHEN updated_at > NOW() - INTERVAL '180 days' THEN 'Last 180d'
|
||||
WHEN updated_at > NOW() - INTERVAL '365 days' THEN 'Last 365d'
|
||||
ELSE 'Older'
|
||||
END AS range,
|
||||
COUNT(*)::int AS skills
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY 1 ORDER BY MIN(NOW() - updated_at)
|
||||
`);
|
||||
report.updatedDist = upRows.rows;
|
||||
table(upRows.rows as Record<string, unknown>[]);
|
||||
|
||||
subHeader('Last Downloaded Distribution');
|
||||
const ldRows = await db.execute(sql`
|
||||
SELECT
|
||||
CASE
|
||||
WHEN last_downloaded_at IS NULL THEN 'Never'
|
||||
WHEN last_downloaded_at > NOW() - INTERVAL '7 days' THEN 'Last 7d'
|
||||
WHEN last_downloaded_at > NOW() - INTERVAL '30 days' THEN 'Last 30d'
|
||||
WHEN last_downloaded_at > NOW() - INTERVAL '90 days' THEN 'Last 90d'
|
||||
ELSE 'Older'
|
||||
END AS range,
|
||||
COUNT(*)::int AS skills
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY 1 ORDER BY MIN(COALESCE(last_downloaded_at, '1970-01-01'::timestamp))
|
||||
`);
|
||||
report.lastDownDist = ldRows.rows;
|
||||
table(ldRows.rows as Record<string, unknown>[]);
|
||||
|
||||
// ======================================================================
|
||||
// 8. TOP OWNERS
|
||||
// ======================================================================
|
||||
header('8. TOP OWNERS');
|
||||
|
||||
subHeader('Top 30 Owners by Skill Count');
|
||||
const owRows = await db.execute(sql`
|
||||
SELECT
|
||||
github_owner AS owner,
|
||||
COUNT(*)::int AS skills,
|
||||
COUNT(DISTINCT github_repo)::int AS repos,
|
||||
MAX(github_stars)::int AS max_stars,
|
||||
COALESCE(SUM(download_count), 0)::int AS downloads,
|
||||
COALESCE(SUM(view_count), 0)::int AS views
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY github_owner ORDER BY skills DESC LIMIT 30
|
||||
`);
|
||||
report.topOwnersByCount = owRows.rows;
|
||||
table(owRows.rows as Record<string, unknown>[]);
|
||||
|
||||
subHeader('Top 20 Owners by Stars');
|
||||
const osRows = await db.execute(sql`
|
||||
SELECT
|
||||
github_owner AS owner,
|
||||
MAX(github_stars)::int AS max_stars,
|
||||
COUNT(*)::int AS skills,
|
||||
COUNT(DISTINCT github_repo)::int AS repos,
|
||||
COALESCE(SUM(download_count), 0)::int AS downloads
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY github_owner ORDER BY max_stars DESC LIMIT 20
|
||||
`);
|
||||
report.topOwnersByStars = osRows.rows;
|
||||
table(osRows.rows as Record<string, unknown>[]);
|
||||
|
||||
subHeader('Owner Concentration');
|
||||
const ocRows = await db.execute(sql`
|
||||
WITH ranked AS (
|
||||
SELECT github_owner, COUNT(*)::int AS cnt,
|
||||
ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY github_owner
|
||||
)
|
||||
SELECT
|
||||
SUM(cnt) FILTER (WHERE rn <= 1)::int AS top_1,
|
||||
SUM(cnt) FILTER (WHERE rn <= 5)::int AS top_5,
|
||||
SUM(cnt) FILTER (WHERE rn <= 10)::int AS top_10,
|
||||
SUM(cnt) FILTER (WHERE rn <= 20)::int AS top_20,
|
||||
SUM(cnt) FILTER (WHERE rn <= 50)::int AS top_50,
|
||||
SUM(cnt)::int AS total,
|
||||
COUNT(*)::int AS owners
|
||||
FROM ranked
|
||||
`);
|
||||
const oc = ocRows.rows[0] as Record<string, unknown>;
|
||||
report.ownerConcentration = oc;
|
||||
|
||||
console.log(` Top 1 owner: ${num(oc.top_1)} skills (${pct(oc.top_1, oc.total)})`);
|
||||
console.log(` Top 5 owners: ${num(oc.top_5)} skills (${pct(oc.top_5, oc.total)})`);
|
||||
console.log(` Top 10 owners: ${num(oc.top_10)} skills (${pct(oc.top_10, oc.total)})`);
|
||||
console.log(` Top 20 owners: ${num(oc.top_20)} skills (${pct(oc.top_20, oc.total)})`);
|
||||
console.log(` Top 50 owners: ${num(oc.top_50)} skills (${pct(oc.top_50, oc.total)})`);
|
||||
console.log(` Total owners: ${num(oc.owners)}`);
|
||||
|
||||
// ======================================================================
|
||||
// 9. MULTI-SKILL REPOS
|
||||
// ======================================================================
|
||||
header('9. MULTI-SKILL REPOS');
|
||||
|
||||
subHeader('Repos with 20+ Skills (Aggregator Candidates)');
|
||||
const aggRows = await db.execute(sql`
|
||||
SELECT
|
||||
github_owner || '/' || github_repo AS repo,
|
||||
COUNT(*)::int AS skills,
|
||||
MAX(github_stars)::int AS stars,
|
||||
COALESCE(SUM(download_count), 0)::int AS downloads
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY github_owner, github_repo
|
||||
HAVING COUNT(*) >= 20
|
||||
ORDER BY skills DESC LIMIT 30
|
||||
`);
|
||||
report.aggregatorCandidates = aggRows.rows;
|
||||
table(aggRows.rows as Record<string, unknown>[]);
|
||||
|
||||
subHeader('Repos with 3-19 Skills (Collection Candidates)');
|
||||
const colRows = await db.execute(sql`
|
||||
SELECT
|
||||
github_owner || '/' || github_repo AS repo,
|
||||
COUNT(*)::int AS skills,
|
||||
MAX(github_stars)::int AS stars,
|
||||
COALESCE(SUM(download_count), 0)::int AS downloads
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY github_owner, github_repo
|
||||
HAVING COUNT(*) BETWEEN 3 AND 19
|
||||
ORDER BY skills DESC LIMIT 30
|
||||
`);
|
||||
report.collectionCandidates = colRows.rows;
|
||||
table(colRows.rows as Record<string, unknown>[]);
|
||||
|
||||
subHeader('Skills-per-Repo Distribution');
|
||||
const sprRows = await db.execute(sql`
|
||||
WITH rc AS (
|
||||
SELECT COUNT(*)::int AS cnt
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY github_owner, github_repo
|
||||
)
|
||||
SELECT
|
||||
CASE
|
||||
WHEN cnt = 1 THEN '1 skill'
|
||||
WHEN cnt = 2 THEN '2 skills'
|
||||
WHEN cnt BETWEEN 3 AND 5 THEN '3-5'
|
||||
WHEN cnt BETWEEN 6 AND 10 THEN '6-10'
|
||||
WHEN cnt BETWEEN 11 AND 20 THEN '11-20'
|
||||
WHEN cnt BETWEEN 21 AND 50 THEN '21-50'
|
||||
WHEN cnt BETWEEN 51 AND 100 THEN '51-100'
|
||||
ELSE '100+'
|
||||
END AS per_repo,
|
||||
COUNT(*)::int AS repos,
|
||||
SUM(cnt)::int AS total_skills
|
||||
FROM rc
|
||||
GROUP BY 1 ORDER BY MIN(cnt)
|
||||
`);
|
||||
report.skillsPerRepo = sprRows.rows;
|
||||
table(sprRows.rows as Record<string, unknown>[]);
|
||||
|
||||
// ======================================================================
|
||||
// 10. CATEGORY DISTRIBUTION
|
||||
// ======================================================================
|
||||
header('10. CATEGORY DISTRIBUTION');
|
||||
|
||||
const catRows = await db.execute(sql`
|
||||
SELECT c.name AS category, c.skill_count::int AS skills, c.id
|
||||
FROM categories c
|
||||
WHERE c.id NOT LIKE 'parent-%'
|
||||
ORDER BY c.skill_count DESC
|
||||
`);
|
||||
report.categoryDist = catRows.rows;
|
||||
table(catRows.rows as Record<string, unknown>[]);
|
||||
|
||||
const ncRows = await db.execute(sql`
|
||||
SELECT COUNT(*)::int AS count
|
||||
FROM skills s
|
||||
WHERE s.is_blocked = false
|
||||
AND NOT EXISTS (SELECT 1 FROM skill_categories sc WHERE sc.skill_id = s.id)
|
||||
`);
|
||||
report.uncategorizedCount = (ncRows.rows[0] as Record<string, unknown>)?.count;
|
||||
console.log(`\n Skills with no category: ${num(report.uncategorizedCount)}`);
|
||||
|
||||
// ======================================================================
|
||||
// 11. FLAGS & RATINGS
|
||||
// ======================================================================
|
||||
header('11. FLAGS & RATINGS');
|
||||
|
||||
const flRows = await db.execute(sql`
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE is_verified = true)::int AS verified,
|
||||
COUNT(*) FILTER (WHERE is_featured = true)::int AS featured,
|
||||
COUNT(*) FILTER (WHERE rating IS NOT NULL)::int AS has_rating,
|
||||
COUNT(*) FILTER (WHERE rating_count > 0)::int AS has_user_ratings,
|
||||
ROUND(AVG(rating) FILTER (WHERE rating IS NOT NULL), 2)::numeric AS avg_rating,
|
||||
MAX(rating_count)::int AS max_rating_count
|
||||
FROM skills WHERE is_blocked = false
|
||||
`);
|
||||
const fl = flRows.rows[0] as Record<string, unknown>;
|
||||
report.flags = fl;
|
||||
|
||||
console.log(` Verified: ${num(fl.verified)}`);
|
||||
console.log(` Featured: ${num(fl.featured)}`);
|
||||
console.log(` Has rating: ${num(fl.has_rating)}`);
|
||||
console.log(` Has user ratings: ${num(fl.has_user_ratings)}`);
|
||||
console.log(` Average rating: ${fl.avg_rating ?? 'N/A'}`);
|
||||
console.log(` Max rating count: ${num(fl.max_rating_count)}`);
|
||||
|
||||
// ======================================================================
|
||||
// 12. CROSS-TABLE STATS
|
||||
// ======================================================================
|
||||
header('12. CROSS-TABLE STATS');
|
||||
|
||||
const xtRows = await db.execute(sql`
|
||||
SELECT
|
||||
(SELECT COUNT(*)::int FROM users) AS users,
|
||||
(SELECT COUNT(*)::int FROM ratings) AS ratings,
|
||||
(SELECT COUNT(*)::int FROM installations) AS installations,
|
||||
(SELECT COUNT(*)::int FROM favorites) AS favorites,
|
||||
(SELECT COUNT(*)::int FROM discovered_repos) AS discovered_repos,
|
||||
(SELECT COUNT(*) FILTER (WHERE has_skill_md = true)::int FROM discovered_repos) AS repos_with_skills,
|
||||
(SELECT COUNT(*)::int FROM removal_requests) AS removal_requests,
|
||||
(SELECT COUNT(*)::int FROM add_requests) AS add_requests,
|
||||
(SELECT COUNT(*) FILTER (WHERE unsubscribed_at IS NULL)::int FROM email_subscriptions) AS subscribers
|
||||
`);
|
||||
const xt = xtRows.rows[0] as Record<string, unknown>;
|
||||
report.crossStats = xt;
|
||||
|
||||
console.log(` Users: ${num(xt.users)}`);
|
||||
console.log(` Ratings: ${num(xt.ratings)}`);
|
||||
console.log(` Installations: ${num(xt.installations)}`);
|
||||
console.log(` Favorites: ${num(xt.favorites)}`);
|
||||
console.log(` Discovered repos: ${num(xt.discovered_repos)}`);
|
||||
console.log(` - with skills: ${num(xt.repos_with_skills)}`);
|
||||
console.log(` Removal requests: ${num(xt.removal_requests)}`);
|
||||
console.log(` Add requests: ${num(xt.add_requests)}`);
|
||||
console.log(` Subscribers: ${num(xt.subscribers)}`);
|
||||
|
||||
subHeader('Installations by Platform');
|
||||
const ipRows = await db.execute(sql`
|
||||
SELECT platform, COUNT(*)::int AS count
|
||||
FROM installations GROUP BY platform ORDER BY count DESC
|
||||
`);
|
||||
report.installByPlatform = ipRows.rows;
|
||||
table(ipRows.rows as Record<string, unknown>[]);
|
||||
|
||||
// ======================================================================
|
||||
// 13. USABILITY SIGNALS
|
||||
// ======================================================================
|
||||
header('13. USABILITY SIGNALS FOR CURATION');
|
||||
|
||||
const usRows = await db.execute(sql`
|
||||
SELECT
|
||||
COUNT(*) FILTER (
|
||||
WHERE LENGTH(description) > 50
|
||||
AND raw_content IS NOT NULL AND LENGTH(raw_content) > 500
|
||||
AND security_status = 'pass'
|
||||
)::int AS strong_standalone,
|
||||
|
||||
COUNT(*) FILTER (
|
||||
WHERE triggers IS NOT NULL AND triggers != '{}'::jsonb
|
||||
AND (triggers->>'filePatterns' IS NOT NULL AND triggers->>'filePatterns' != '[]')
|
||||
)::int AS has_file_triggers,
|
||||
|
||||
COUNT(*) FILTER (WHERE COALESCE(download_count, 0) > 0)::int AS ever_downloaded,
|
||||
|
||||
COUNT(*) FILTER (
|
||||
WHERE github_stars >= 10
|
||||
AND LENGTH(description) > 50
|
||||
AND raw_content IS NOT NULL
|
||||
AND security_status = 'pass'
|
||||
)::int AS high_quality,
|
||||
|
||||
COUNT(*) FILTER (
|
||||
WHERE github_stars >= 100
|
||||
AND COALESCE(download_count, 0) > 0
|
||||
AND security_status = 'pass'
|
||||
)::int AS premium,
|
||||
|
||||
COUNT(*) FILTER (
|
||||
WHERE source_format = 'skill.md'
|
||||
AND LENGTH(description) > 50
|
||||
AND raw_content IS NOT NULL AND LENGTH(raw_content) > 300
|
||||
AND security_status = 'pass'
|
||||
)::int AS skillmd_quality
|
||||
FROM skills WHERE is_blocked = false
|
||||
`);
|
||||
const us = usRows.rows[0] as Record<string, unknown>;
|
||||
report.usability = us;
|
||||
|
||||
console.log(` Strong standalone candidates: ${num(us.strong_standalone)}`);
|
||||
console.log(` (desc>50 + content>500 + security=pass)`);
|
||||
console.log(` With file triggers: ${num(us.has_file_triggers)}`);
|
||||
console.log(` Ever downloaded: ${num(us.ever_downloaded)}`);
|
||||
console.log(` High quality (stars>10+desc+sec): ${num(us.high_quality)}`);
|
||||
console.log(` Premium (stars>100+dl>0+sec): ${num(us.premium)}`);
|
||||
console.log(` SKILL.md with quality: ${num(us.skillmd_quality)}`);
|
||||
|
||||
// ======================================================================
|
||||
// 14. STANDALONE vs PROJECT-BOUND
|
||||
// ======================================================================
|
||||
header('14. STANDALONE vs PROJECT-BOUND HEURISTIC');
|
||||
|
||||
subHeader('Single vs Multi-Skill Repos');
|
||||
const smRows = await db.execute(sql`
|
||||
WITH rc AS (
|
||||
SELECT github_owner, github_repo, COUNT(*)::int AS cnt
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY github_owner, github_repo
|
||||
)
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE cnt = 1)::int AS single,
|
||||
COUNT(*) FILTER (WHERE cnt = 2)::int AS two,
|
||||
COUNT(*) FILTER (WHERE cnt >= 3)::int AS multi,
|
||||
COUNT(*)::int AS total
|
||||
FROM rc
|
||||
`);
|
||||
const sm = smRows.rows[0] as Record<string, unknown>;
|
||||
report.repoTypes = sm;
|
||||
|
||||
console.log(` Single-skill repos: ${num(sm.single)} (${pct(sm.single, sm.total)})`);
|
||||
console.log(` Two-skill repos: ${num(sm.two)} (${pct(sm.two, sm.total)})`);
|
||||
console.log(` Multi-skill repos: ${num(sm.multi)} (${pct(sm.multi, sm.total)})`);
|
||||
|
||||
subHeader('Name Pattern Signals');
|
||||
const npRows = await db.execute(sql`
|
||||
SELECT
|
||||
CASE
|
||||
WHEN name ILIKE '%rule%' OR name ILIKE '%config%' OR name ILIKE '%setup%' THEN 'rules/config/setup'
|
||||
WHEN name ILIKE '%my-%' OR name ILIKE '%my_%' THEN 'starts with my-'
|
||||
WHEN name ILIKE '%cursor%' OR name ILIKE '%claude%' OR name ILIKE '%copilot%' THEN 'tool-specific'
|
||||
WHEN name ILIKE '%project%' OR name ILIKE '%team%' OR name ILIKE '%internal%' THEN 'project/team'
|
||||
WHEN name ILIKE '%.mdc' OR name ILIKE '%cursorrule%' THEN 'cursor rules file'
|
||||
ELSE 'generic/other'
|
||||
END AS pattern,
|
||||
COUNT(*)::int AS skills
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY 1 ORDER BY skills DESC
|
||||
`);
|
||||
report.namePatterns = npRows.rows;
|
||||
table(npRows.rows as Record<string, unknown>[]);
|
||||
|
||||
// ======================================================================
|
||||
// 15. SAMPLE SKILLS
|
||||
// ======================================================================
|
||||
header('15. SAMPLE SKILLS');
|
||||
|
||||
subHeader('Top 15 by Stars');
|
||||
const tsRows = await db.execute(sql`
|
||||
SELECT id, name, github_stars AS stars, COALESCE(download_count,0) AS dl,
|
||||
security_status AS sec, source_format AS fmt,
|
||||
LEFT(description, 80) AS description
|
||||
FROM skills WHERE is_blocked = false
|
||||
ORDER BY github_stars DESC LIMIT 15
|
||||
`);
|
||||
report.topByStars = tsRows.rows;
|
||||
table(tsRows.rows as Record<string, unknown>[]);
|
||||
|
||||
subHeader('Top 15 by Downloads (actually used)');
|
||||
const tdRows = await db.execute(sql`
|
||||
SELECT id, name, download_count AS dl, view_count AS views,
|
||||
github_stars AS stars, security_status AS sec,
|
||||
LEFT(description, 80) AS description
|
||||
FROM skills WHERE is_blocked = false AND COALESCE(download_count,0) > 0
|
||||
ORDER BY download_count DESC LIMIT 15
|
||||
`);
|
||||
report.topByDL = tdRows.rows;
|
||||
table(tdRows.rows as Record<string, unknown>[]);
|
||||
|
||||
subHeader('SKILL.md + Stars>50 + Downloads>0');
|
||||
const qsRows = await db.execute(sql`
|
||||
SELECT id, name, github_stars AS stars, download_count AS dl,
|
||||
security_status AS sec, LEFT(description, 80) AS description
|
||||
FROM skills
|
||||
WHERE is_blocked = false AND source_format = 'skill.md'
|
||||
AND github_stars >= 50 AND COALESCE(download_count, 0) > 0
|
||||
ORDER BY github_stars DESC LIMIT 20
|
||||
`);
|
||||
report.qualitySkillMd = qsRows.rows;
|
||||
table(qsRows.rows as Record<string, unknown>[]);
|
||||
|
||||
// ======================================================================
|
||||
// 16. DISCOVERED REPOS
|
||||
// ======================================================================
|
||||
header('16. DISCOVERED REPOS');
|
||||
|
||||
const drRows = await db.execute(sql`
|
||||
SELECT
|
||||
discovered_via AS source,
|
||||
COUNT(*)::int AS total,
|
||||
COUNT(*) FILTER (WHERE has_skill_md = true)::int AS with_skills,
|
||||
COUNT(*) FILTER (WHERE last_scanned IS NOT NULL)::int AS scanned,
|
||||
COUNT(*) FILTER (WHERE is_archived = true)::int AS archived
|
||||
FROM discovered_repos
|
||||
GROUP BY discovered_via ORDER BY total DESC
|
||||
`);
|
||||
report.discoveredStats = drRows.rows;
|
||||
table(drRows.rows as Record<string, unknown>[]);
|
||||
|
||||
// ======================================================================
|
||||
// 17. CACHED FILES
|
||||
// ======================================================================
|
||||
header('17. CACHED FILES');
|
||||
|
||||
const cfRows = await db.execute(sql`
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE cached_files IS NOT NULL)::int AS has_cache,
|
||||
COUNT(*) FILTER (WHERE cached_files IS NULL)::int AS no_cache
|
||||
FROM skills WHERE is_blocked = false
|
||||
`);
|
||||
const cf = cfRows.rows[0] as Record<string, unknown>;
|
||||
report.cacheStats = cf;
|
||||
|
||||
console.log(` Has cached files: ${num(cf.has_cache)} (${pct(cf.has_cache, total)})`);
|
||||
console.log(` No cached files: ${num(cf.no_cache)} (${pct(cf.no_cache, total)})`);
|
||||
|
||||
// ======================================================================
|
||||
// 19. BRANCH DISTRIBUTION
|
||||
// ======================================================================
|
||||
header('19. BRANCH DISTRIBUTION');
|
||||
|
||||
const brRows = await db.execute(sql`
|
||||
SELECT COALESCE(branch, 'null') AS branch, COUNT(*)::int AS count
|
||||
FROM skills WHERE is_blocked = false
|
||||
GROUP BY branch ORDER BY count DESC LIMIT 10
|
||||
`);
|
||||
report.branchDist = brRows.rows;
|
||||
table(brRows.rows as Record<string, unknown>[]);
|
||||
|
||||
// ======================================================================
|
||||
// 20. DUPLICATES
|
||||
// ======================================================================
|
||||
header('20. POTENTIAL DUPLICATES');
|
||||
|
||||
subHeader('Same Name + Description');
|
||||
const ddRows = await db.execute(sql`
|
||||
SELECT name, LEFT(description, 60) AS desc, COUNT(*)::int AS occurrences,
|
||||
STRING_AGG(DISTINCT github_owner, ', ') AS owners
|
||||
FROM skills
|
||||
WHERE is_blocked = false AND description IS NOT NULL AND LENGTH(description) > 10
|
||||
GROUP BY name, description HAVING COUNT(*) > 1
|
||||
ORDER BY occurrences DESC LIMIT 15
|
||||
`);
|
||||
report.exactDupes = ddRows.rows;
|
||||
table(ddRows.rows as Record<string, unknown>[]);
|
||||
|
||||
subHeader('Same Content Hash (identical content, 3+ copies)');
|
||||
const chRows = await db.execute(sql`
|
||||
SELECT content_hash, COUNT(*)::int AS copies,
|
||||
MIN(name) AS sample_name,
|
||||
STRING_AGG(DISTINCT github_owner, ', ') AS owners
|
||||
FROM skills
|
||||
WHERE is_blocked = false AND content_hash IS NOT NULL
|
||||
GROUP BY content_hash HAVING COUNT(*) > 2
|
||||
ORDER BY copies DESC LIMIT 15
|
||||
`);
|
||||
report.hashDupes = chRows.rows;
|
||||
table(chRows.rows as Record<string, unknown>[]);
|
||||
|
||||
// ======================================================================
|
||||
// DONE
|
||||
// ======================================================================
|
||||
const duration = ((Date.now() - startTime) / 1000).toFixed(1);
|
||||
header(`EXPLORATION COMPLETE (${duration}s)`);
|
||||
|
||||
console.log(`
|
||||
KEY QUESTIONS TO ANSWER:
|
||||
1. What % of skills are SKILL.md vs other formats?
|
||||
2. What % have been downloaded at least once? (real usage)
|
||||
3. How concentrated is ownership? (top 10 owners = ?%)
|
||||
4. How many multi-skill repos? (collection/aggregator candidates)
|
||||
5. How many pass security + have good content? (curation-ready)
|
||||
6. How many duplicates exist?
|
||||
`);
|
||||
|
||||
// Save JSON report
|
||||
report.generatedAt = new Date().toISOString();
|
||||
report.durationSeconds = parseFloat(duration);
|
||||
|
||||
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
||||
const reportPath = resolve(scriptDir, 'explore-report.json');
|
||||
writeFileSync(reportPath, JSON.stringify(report, null, 2), 'utf-8');
|
||||
console.log(` Full report saved to: ${reportPath}`);
|
||||
|
||||
await closeDb();
|
||||
}
|
||||
|
||||
// ─── Run ───────────────────────────────────────────────────────────────────────
|
||||
runExploration().catch(async (err) => {
|
||||
console.error('Exploration failed:', err);
|
||||
await closeDb();
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user