atlatl/skillhub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sync: malware page, security alerts, review diagnostics, skill detail improvements

Browse Source 
- Add malware security advisory page and SecurityAlertBanner component
- Add review diagnostics API and reviewed stats page
- Improve skill detail page with better scoring display and metadata
- Update review API endpoints with enhanced filtering and stats
- Add skill file serving improvements and cache enhancements
- Remove legacy curation scripts (moved to internal tooling)
- Update CLI search with score filtering support

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
airano-ir
2026-03-31 16:08:12 +02:00
parent 3e22d2e238
commit 16ee9e3beb
33 changed files with 1231 additions and 3404 deletions
Download Patch File Download Diff File Expand all files Collapse all files

458
scripts/curation/batch-score.mjs
View File

@@ -1,458 +0,0 @@
#!/usr/bin/env node
/**
* Phase 5: Batch Quality Score
*
* Calculates quality scores for browse-ready skills using the same logic
* as SkillAnalyzer.calculateQuality() from services/indexer/src/analyzer.ts.
*
* Updates: quality_score, quality_details, review_status → 'auto-scored'
*
* Usage:
* DATABASE_URL=postgres://... node scripts/curation/batch-score.mjs
*
* Options:
* --dry-run Show what would change without writing
* --batch-size=N Process N skills per batch (default 100)
*/
import { createRequire } from 'module';
import { resolve, dirname } from 'path';
import { fileURLToPath, pathToFileURL } from 'url';
const __dirname = dirname(fileURLToPath(import.meta.url));
// ─── Find pg module ───
let pg;
const tryPaths = [
resolve(__dirname, '../..', 'package.json'),
'/tmp/package.json',
process.env.APPDATA ? resolve(process.env.APPDATA, '..', 'Local', 'Temp', 'package.json') : null,
].filter(Boolean);
for (const p of tryPaths) {
try { pg = createRequire(p)('pg'); break; } catch {}
}
if (!pg) { console.error('pg not found. Run: npm install pg'); process.exit(1); }
// ─── Find skillhub-core (ESM-only — use direct path import) ───
let parseSkillMd, parseGenericInstructionFile, validateSkill, scanSecurity;
const corePaths = [
resolve(__dirname, '../../packages/core/dist/index.js'),
resolve(__dirname, '../../node_modules/skillhub-core/dist/index.js'),
resolve(__dirname, '../../services/indexer/node_modules/skillhub-core/dist/index.js'),
];
for (const p of corePaths) {
try {
const core = await import(pathToFileURL(p).href);
parseSkillMd = core.parseSkillMd;
parseGenericInstructionFile = core.parseGenericInstructionFile;
validateSkill = core.validateSkill;
scanSecurity = core.scanSecurity;
break;
} catch {}
}
if (!parseSkillMd) { console.error('skillhub-core not found. Run: pnpm build'); process.exit(1); }
// ─── Config ───
const DATABASE_URL = process.env.DATABASE_URL || 'postgresql://postgres:postgres@localhost:5432/skillhub';
const DRY_RUN = process.argv.includes('--dry-run');
const batchArg = process.argv.find(a => a.startsWith('--batch-size='));
const BATCH_SIZE = batchArg ? parseInt(batchArg.split('=')[1]) : 100;
// ─── Helpers ───
const n = v => Number(v ?? 0).toLocaleString('en-US');
function progress(current, total) {
const pct = ((current / total) * 100).toFixed(1);
process.stdout.write(`\r Processing ${n(current)} / ${n(total)} (${pct}%)`);
}
// ─── Database ───
let client;
async function connect() {
client = new pg.Client({
connectionString: DATABASE_URL,
ssl: false,
connectionTimeoutMillis: 15000,
query_timeout: 600000,
keepAlive: true,
});
await client.connect();
console.log('Connected to database');
}
async function query(sql, params = []) {
return client.query(sql, params);
}
// ─── Quality scoring (mirrors SkillAnalyzer.calculateQuality) ───
function scoreDocumentation(skill, scripts, references) {
let score = 0;
// Has description (required)
if (skill.metadata.description && skill.metadata.description.length > 20) {
score += 20;
}
// Content length and structure
const contentLength = skill.content.length;
if (contentLength > 500) score += 15;
else if (contentLength > 200) score += 10;
else if (contentLength > 50) score += 5;
// Has headers (good structure)
const headerCount = (skill.content.match(/^#+\s/gm) || []).length;
if (headerCount >= 3) score += 15;
else if (headerCount >= 1) score += 10;
// Has code examples
if (skill.content.includes('```')) {
score += 15;
}
// Has version
if (skill.metadata.version) {
score += 10;
}
// Has license
if (skill.metadata.license) {
score += 5;
}
// Has compatibility info
if (skill.metadata.compatibility?.platforms?.length) {
score += 10;
}
// Has scripts
if (scripts.length > 0) {
score += 5;
}
// Has references
if (references.length > 0) {
score += 5;
}
return Math.min(100, score);
}
function scoreMaintenance(repoMeta) {
let score = 0;
// Check last update time
const lastUpdate = new Date(repoMeta.updatedAt);
const daysSinceUpdate = (Date.now() - lastUpdate.getTime()) / (1000 * 60 * 60 * 24);
if (daysSinceUpdate < 30) score += 40;
else if (daysSinceUpdate < 90) score += 30;
else if (daysSinceUpdate < 180) score += 20;
else if (daysSinceUpdate < 365) score += 10;
// Has license
if (repoMeta.license) {
score += 20;
}
// Has description
if (repoMeta.description) {
score += 10;
}
// Has topics (always [] since not in DB — will miss ~10 points)
if (repoMeta.topics.length > 0) {
score += 10;
}
// Activity level (forks indicate usage)
if (repoMeta.forks >= 10) score += 20;
else if (repoMeta.forks >= 5) score += 15;
else if (repoMeta.forks >= 1) score += 10;
return Math.min(100, score);
}
function scorePopularity(repoMeta) {
const stars = repoMeta.stars;
const forks = repoMeta.forks;
let score = 0;
if (stars >= 1000) score += 50;
else if (stars >= 100) score += 40;
else if (stars >= 50) score += 30;
else if (stars >= 10) score += 20;
else if (stars >= 5) score += 10;
else if (stars >= 1) score += 5;
if (forks >= 50) score += 30;
else if (forks >= 10) score += 20;
else if (forks >= 5) score += 15;
else if (forks >= 1) score += 10;
// Bonus for relevant topics (always [] from DB — will miss ~20 points)
const relevantTopics = ['ai', 'agent', 'skill', 'claude', 'copilot', 'codex', 'llm'];
const hasRelevantTopic = repoMeta.topics.some(t =>
relevantTopics.some(rt => t.toLowerCase().includes(rt))
);
if (hasRelevantTopic) {
score += 20;
}
return Math.min(100, score);
}
function calculateQuality(skill, repoMeta, securityScore, validation, scripts, references) {
const factors = [];
// Documentation quality (30% weight)
const docScore = scoreDocumentation(skill, scripts, references);
factors.push({ name: 'documentation', score: docScore, weight: 0.3 });
// Maintenance signals (25% weight)
const maintScore = scoreMaintenance(repoMeta);
factors.push({ name: 'maintenance', score: maintScore, weight: 0.25 });
// Popularity (20% weight)
const popScore = scorePopularity(repoMeta);
factors.push({ name: 'popularity', score: popScore, weight: 0.2 });
// Security (15% weight)
factors.push({ name: 'security', score: securityScore, weight: 0.15 });
// Validation (10% weight)
const valScore = validation.isValid ? 100 : Math.max(0, 100 - validation.errors.length * 20);
factors.push({ name: 'validation', score: valScore, weight: 0.1 });
// Calculate weighted overall score
const overall = Math.round(
factors.reduce((sum, f) => sum + f.score * f.weight, 0)
);
return {
overall,
documentation: docScore,
maintenance: maintScore,
popularity: popScore,
factors,
};
}
// ─── Extract scripts/references from cached_files ───
function extractScripts(cachedFiles) {
if (!cachedFiles || !cachedFiles.items) return [];
return cachedFiles.items
.filter(item =>
!item.isBinary &&
item.name !== 'SKILL.md' &&
(item.name.endsWith('.sh') ||
item.name.endsWith('.py') ||
item.name.endsWith('.js') ||
item.name.endsWith('.ts') ||
item.name.endsWith('.ps1') ||
item.name.endsWith('.bat') ||
item.name.endsWith('.rb'))
)
.map(item => ({ name: item.name, content: item.content }));
}
function extractReferences(cachedFiles) {
if (!cachedFiles || !cachedFiles.items) return [];
return cachedFiles.items
.filter(item =>
!item.isBinary &&
item.name !== 'SKILL.md' &&
(item.name.endsWith('.md') || item.name.endsWith('.txt'))
)
.map(item => ({ name: item.name, content: item.content }));
}
// ─── Main ───
async function main() {
const t0 = Date.now();
await connect();
console.log(`\n${'='.repeat(70)}`);
console.log(' BATCH QUALITY SCORE (Phase 5)');
console.log(`${'='.repeat(70)}`);
if (DRY_RUN) console.log('\n *** DRY RUN MODE — no changes will be made ***\n');
// Count skills to score
const countResult = await query(`
SELECT COUNT(*)::int AS total
FROM skills
WHERE is_blocked = false
AND is_duplicate = false
AND quality_score IS NULL
AND raw_content IS NOT NULL
`);
const total = countResult.rows[0].total;
console.log(` Skills to score: ${n(total)}`);
console.log(` Batch size: ${BATCH_SIZE}`);
if (total === 0) {
console.log(' Nothing to do — all browse-ready skills already scored');
await client.end().catch(() => {});
return;
}
if (DRY_RUN) {
const sample = await query(`
SELECT id, LEFT(name, 50) AS name, github_stars
FROM skills
WHERE is_blocked = false
AND is_duplicate = false
AND is_duplicate = false
AND quality_score IS NULL
AND raw_content IS NOT NULL
ORDER BY github_stars DESC NULLS LAST
LIMIT 5
`);
console.log('\n Sample skills that would be scored:');
for (const r of sample.rows) {
console.log(` [${n(r.github_stars)}★] ${r.id} — ${r.name}`);
}
console.log(`\n [DRY RUN] Would score ${n(total)} skills`);
await client.end().catch(() => {});
return;
}
// Process in batches
let processed = 0;
let errorCount = 0;
let totalScore = 0;
const scoreBuckets = { excellent: 0, good: 0, fair: 0, poor: 0 };
while (processed < total) {
const batch = await query(`
SELECT id, raw_content, cached_files, source_format,
github_stars, github_forks, license, description,
updated_at, version, compatibility, security_score
FROM skills
WHERE is_blocked = false
AND is_duplicate = false
AND is_duplicate = false
AND quality_score IS NULL
AND raw_content IS NOT NULL
ORDER BY github_stars DESC NULLS LAST
LIMIT $1
`, [BATCH_SIZE]);
if (batch.rows.length === 0) break;
for (const row of batch.rows) {
try {
// Parse the skill content
const sourceFormat = row.source_format || 'skill.md';
let skill;
if (sourceFormat === 'skill.md') {
skill = parseSkillMd(row.raw_content);
} else {
skill = parseGenericInstructionFile(row.raw_content, sourceFormat, {
name: row.description?.split(/\s+/).slice(0, 3).join('-') || 'skill',
description: row.description,
owner: '',
});
}
// Validate
const validation = sourceFormat === 'skill.md'
? validateSkill(skill)
: { isValid: true, errors: [], warnings: [] };
// Construct pseudo-repoMeta from DB fields
const repoMeta = {
stars: row.github_stars || 0,
forks: row.github_forks || 0,
license: row.license || null,
description: row.description || null,
updatedAt: row.updated_at ? row.updated_at.toISOString() : new Date().toISOString(),
defaultBranch: 'main',
topics: [], // NOT in DB — popularity factor will miss ~20 points
};
// Use already-computed security score (from Phase 4), default to 100 if not scanned
const securityScore = row.security_score ?? 100;
// Extract scripts/references from cached_files
const scripts = extractScripts(row.cached_files);
const references = extractReferences(row.cached_files);
// Calculate quality
const quality = calculateQuality(skill, repoMeta, securityScore, validation, scripts, references);
// Build quality_details for JSONB storage
const qualityDetails = {
documentation: quality.documentation,
maintenance: quality.maintenance,
popularity: quality.popularity,
factors: quality.factors,
};
await query(`
UPDATE skills
SET quality_score = $1,
quality_details = $2,
review_status = CASE
WHEN review_status IS NULL OR review_status = 'unreviewed'
THEN 'auto-scored'
ELSE review_status
END
WHERE id = $3
`, [quality.overall, JSON.stringify(qualityDetails), row.id]);
totalScore += quality.overall;
if (quality.overall >= 70) scoreBuckets.excellent++;
else if (quality.overall >= 50) scoreBuckets.good++;
else if (quality.overall >= 30) scoreBuckets.fair++;
else scoreBuckets.poor++;
} catch (err) {
errorCount++;
if (errorCount <= 5) {
console.error(`\n Error scoring ${row.id}: ${err.message}`);
}
}
processed++;
if (processed % 100 === 0 || processed === total) {
progress(processed, total);
}
}
}
console.log('\n'); // Clear progress line
// Summary
const scored = processed - errorCount;
const avgScore = scored > 0 ? (totalScore / scored).toFixed(1) : 0;
console.log(`
┌─────────────────────────────────────────────────────┐
│ QUALITY SCORE SUMMARY │
├─────────────────────────────────────────────────────┤
│ Total scored: ${n(scored).padStart(8)} │
│ Errors (skipped): ${n(errorCount).padStart(8)} │
├─────────────────────────────────────────────────────┤
│ Excellent (70+): ${n(scoreBuckets.excellent).padStart(8)} │
│ Good (50-69): ${n(scoreBuckets.good).padStart(8)} │
│ Fair (30-49): ${n(scoreBuckets.fair).padStart(8)} │
│ Poor (<30): ${n(scoreBuckets.poor).padStart(8)} │
├─────────────────────────────────────────────────────┤
│ Avg quality score: ${avgScore.toString().padStart(8)} │
└─────────────────────────────────────────────────────┘
`);
console.log(' Note: repoMeta.topics not in DB — popularity score misses ~20 points');
const dur = ((Date.now() - t0) / 1000).toFixed(1);
console.log(`\nCompleted in ${dur}s`);
await client.end().catch(() => {});
}
main().catch(async e => {
console.error('FAILED:', e.message || e);
await client?.end().catch(() => {});
process.exit(1);
});

239
scripts/curation/batch-security.mjs
View File

@@ -1,239 +0,0 @@
#!/usr/bin/env node
/**
* Phase 4: Batch Security Scan
*
* Scans browse-ready skills for security issues using scanSecurity() from skillhub-core.
* Updates: security_score, security_status, last_scanned, review_status → 'auto-scored'
*
* Usage:
* DATABASE_URL=postgres://... node scripts/curation/batch-security.mjs
*
* Options:
* --dry-run Show what would change without writing
* --batch-size=N Process N skills per batch (default 100)
*/
import { createRequire } from 'module';
import { resolve, dirname } from 'path';
import { fileURLToPath, pathToFileURL } from 'url';
const __dirname = dirname(fileURLToPath(import.meta.url));
// ─── Find pg module ───
let pg;
const tryPaths = [
resolve(__dirname, '../..', 'package.json'),
'/tmp/package.json',
process.env.APPDATA ? resolve(process.env.APPDATA, '..', 'Local', 'Temp', 'package.json') : null,
].filter(Boolean);
for (const p of tryPaths) {
try { pg = createRequire(p)('pg'); break; } catch {}
}
if (!pg) { console.error('pg not found. Run: npm install pg'); process.exit(1); }
// ─── Find skillhub-core (ESM-only — use direct path import) ───
let scanSecurity;
const corePaths = [
resolve(__dirname, '../../packages/core/dist/index.js'),
resolve(__dirname, '../../node_modules/skillhub-core/dist/index.js'),
resolve(__dirname, '../../services/indexer/node_modules/skillhub-core/dist/index.js'),
];
for (const p of corePaths) {
try {
const core = await import(pathToFileURL(p).href);
scanSecurity = core.scanSecurity;
break;
} catch {}
}
if (!scanSecurity) { console.error('skillhub-core not found. Run: pnpm build'); process.exit(1); }
// ─── Config ───
const DATABASE_URL = process.env.DATABASE_URL || 'postgresql://postgres:postgres@localhost:5432/skillhub';
const DRY_RUN = process.argv.includes('--dry-run');
const batchArg = process.argv.find(a => a.startsWith('--batch-size='));
const BATCH_SIZE = batchArg ? parseInt(batchArg.split('=')[1]) : 100;
// ─── Helpers ───
const n = v => Number(v ?? 0).toLocaleString('en-US');
function progress(current, total) {
const pct = ((current / total) * 100).toFixed(1);
process.stdout.write(`\r Processing ${n(current)} / ${n(total)} (${pct}%)`);
}
// ─── Database ───
let client;
async function connect() {
client = new pg.Client({
connectionString: DATABASE_URL,
ssl: false,
connectionTimeoutMillis: 15000,
query_timeout: 600000,
keepAlive: true,
});
await client.connect();
console.log('Connected to database');
}
async function query(sql, params = []) {
return client.query(sql, params);
}
// ─── Extract scripts from cached_files ───
function extractScripts(cachedFiles) {
if (!cachedFiles || !cachedFiles.items) return [];
return cachedFiles.items
.filter(item =>
!item.isBinary &&
item.name !== 'SKILL.md' &&
(item.name.endsWith('.sh') ||
item.name.endsWith('.py') ||
item.name.endsWith('.js') ||
item.name.endsWith('.ts') ||
item.name.endsWith('.ps1') ||
item.name.endsWith('.bat') ||
item.name.endsWith('.rb'))
)
.map(item => ({ name: item.name, content: item.content }));
}
// ─── Main ───
async function main() {
const t0 = Date.now();
await connect();
console.log(`\n${'='.repeat(70)}`);
console.log(' BATCH SECURITY SCAN (Phase 4)');
console.log(`${'='.repeat(70)}`);
if (DRY_RUN) console.log('\n *** DRY RUN MODE — no changes will be made ***\n');
// Count skills to scan
const countResult = await query(`
SELECT COUNT(*)::int AS total
FROM skills
WHERE is_blocked = false
AND is_duplicate = false
AND security_status IS NULL
AND raw_content IS NOT NULL
`);
const total = countResult.rows[0].total;
console.log(` Skills to scan: ${n(total)}`);
console.log(` Batch size: ${BATCH_SIZE}`);
if (total === 0) {
console.log(' Nothing to do — all browse-ready skills already scanned');
await client.end().catch(() => {});
return;
}
if (DRY_RUN) {
// Show sample
const sample = await query(`
SELECT id, LEFT(name, 50) AS name
FROM skills
WHERE is_blocked = false
AND is_duplicate = false
AND is_duplicate = false
AND security_status IS NULL
AND raw_content IS NOT NULL
LIMIT 5
`);
console.log('\n Sample skills that would be scanned:');
for (const r of sample.rows) {
console.log(` ${r.id} — ${r.name}`);
}
console.log(`\n [DRY RUN] Would scan ${n(total)} skills`);
await client.end().catch(() => {});
return;
}
// Process in batches
let processed = 0;
let passCount = 0;
let warnCount = 0;
let failCount = 0;
let errorCount = 0;
let totalScore = 0;
while (processed < total) {
const batch = await query(`
SELECT id, raw_content, cached_files
FROM skills
WHERE is_blocked = false
AND is_duplicate = false
AND is_duplicate = false
AND security_status IS NULL
AND raw_content IS NOT NULL
ORDER BY github_stars DESC NULLS LAST
LIMIT $1
`, [BATCH_SIZE]);
if (batch.rows.length === 0) break;
for (const row of batch.rows) {
try {
const scripts = extractScripts(row.cached_files);
const report = scanSecurity({ content: row.raw_content, scripts });
await query(`
UPDATE skills
SET security_score = $1,
security_status = $2,
last_scanned = NOW(),
review_status = CASE
WHEN review_status IS NULL OR review_status = 'unreviewed'
THEN 'auto-scored'
ELSE review_status
END
WHERE id = $3
`, [report.score, report.status, row.id]);
totalScore += report.score;
if (report.status === 'pass') passCount++;
else if (report.status === 'warning') warnCount++;
else failCount++;
} catch (err) {
errorCount++;
if (errorCount <= 5) {
console.error(`\n Error scanning ${row.id}: ${err.message}`);
}
}
processed++;
if (processed % 100 === 0 || processed === total) {
progress(processed, total);
}
}
}
console.log('\n'); // Clear progress line
// Summary
const avgScore = processed > 0 ? (totalScore / (processed - errorCount)).toFixed(1) : 0;
console.log(`
┌─────────────────────────────────────────────────────┐
│ SECURITY SCAN SUMMARY │
├─────────────────────────────────────────────────────┤
│ Total scanned: ${n(processed).padStart(8)} │
│ Errors (skipped): ${n(errorCount).padStart(8)} │
├─────────────────────────────────────────────────────┤
│ PASS: ${n(passCount).padStart(8)} │
│ WARNING: ${n(warnCount).padStart(8)} │
│ FAIL: ${n(failCount).padStart(8)} │
├─────────────────────────────────────────────────────┤
│ Avg security score:${avgScore.toString().padStart(8)} │
└─────────────────────────────────────────────────────┘
`);
const dur = ((Date.now() - t0) / 1000).toFixed(1);
console.log(`Completed in ${dur}s`);
await client.end().catch(() => {});
}
main().catch(async e => {
console.error('FAILED:', e.message || e);
await client?.end().catch(() => {});
process.exit(1);
});

505
scripts/curation/curate.mjs
View File

@@ -1,505 +0,0 @@
#!/usr/bin/env node
/**
* Phase 2: Data Cleanup & Classification
*
* Runs all curation steps in order:
* Step 1: Compute repo_skill_count for every skill
* Step 2: Mark aggregators (repos with 50+ skills)
* Step 3: Classify remaining repos (collection / standalone / project-bound)
* Step 4: Fill missing content_hash values
* Step 5: Mark duplicates by content_hash (canonical = highest stars or oldest)
* Step 6: Detect fork marketplace repos
* Step 7: Recalculate category skill counts (browse-ready only)
* Step 8: Summary report
*
* Usage:
* DATABASE_URL=postgres://... node scripts/curation/curate.mjs
* # Or with local Docker:
* DATABASE_URL=postgresql://postgres:postgres@localhost:5432/skillhub node scripts/curation/curate.mjs
*
* Options:
* --dry-run Show what would change without writing
* --step=N Run only step N (1-8)
*/
import { createRequire } from 'module';
import { resolve, dirname } from 'path';
import { fileURLToPath } from 'url';
const __dirname = dirname(fileURLToPath(import.meta.url));
// ─── Find pg module ───
let pg;
const tryPaths = [
resolve(__dirname, '../..', 'package.json'),
'/tmp/package.json',
process.env.APPDATA ? resolve(process.env.APPDATA, '..', 'Local', 'Temp', 'package.json') : null,
].filter(Boolean);
for (const p of tryPaths) {
try { pg = createRequire(p)('pg'); break; } catch {}
}
if (!pg) { console.error('pg not found. Run: npm install pg'); process.exit(1); }
// ─── Config ───
const DATABASE_URL = process.env.DATABASE_URL || 'postgresql://postgres:postgres@localhost:5432/skillhub';
const DRY_RUN = process.argv.includes('--dry-run');
const stepArg = process.argv.find(a => a.startsWith('--step='));
const ONLY_STEP = stepArg ? parseInt(stepArg.split('=')[1]) : null;
// ─── Helpers ───
function header(step, title) {
console.log(`\n${'='.repeat(70)}\n STEP ${step}: ${title}\n${'='.repeat(70)}`);
}
const n = v => Number(v ?? 0).toLocaleString('en-US');
// ─── Database ───
let client;
async function connect() {
client = new pg.Client({
connectionString: DATABASE_URL,
ssl: false,
connectionTimeoutMillis: 15000,
query_timeout: 600000, // 10 min for big updates
keepAlive: true,
});
await client.connect();
console.log('Connected to database');
}
async function query(sql, params = []) {
const result = await client.query(sql, params);
return result;
}
// ─── Step 1: Compute repo_skill_count ───
async function step1_repoSkillCount() {
header(1, 'COMPUTE repo_skill_count');
const countResult = await query(`
SELECT COUNT(*)::int AS total
FROM skills
WHERE is_blocked = false AND repo_skill_count IS NULL
`);
console.log(` Skills without repo_skill_count: ${n(countResult.rows[0].total)}`);
if (DRY_RUN) { console.log(' [DRY RUN] Would update all skills'); return; }
const result = await query(`
UPDATE skills s SET repo_skill_count = sub.cnt
FROM (
SELECT github_owner, github_repo, COUNT(*)::int AS cnt
FROM skills WHERE is_blocked = false
GROUP BY github_owner, github_repo
) sub
WHERE s.github_owner = sub.github_owner
AND s.github_repo = sub.github_repo
AND s.is_blocked = false
`);
console.log(` Updated: ${n(result.rowCount)} skills`);
}
// ─── Step 2: Mark aggregators ───
async function step2_markAggregators() {
header(2, 'MARK AGGREGATORS (repos with 50+ skills)');
// Preview
const preview = await query(`
SELECT github_owner || '/' || github_repo AS repo, COUNT(*)::int AS skills,
MAX(github_stars)::int AS stars
FROM skills WHERE is_blocked = false
GROUP BY github_owner, github_repo
HAVING COUNT(*) >= 50
ORDER BY skills DESC LIMIT 20
`);
console.log(` Top aggregator repos (${preview.rowCount} total):`);
for (const r of preview.rows.slice(0, 10)) {
console.log(` ${r.repo}: ${n(r.skills)} skills (${n(r.stars)} stars)`);
}
if (DRY_RUN) { console.log(' [DRY RUN] Would mark skills in these repos'); return; }
const result = await query(`
UPDATE skills s SET skill_type = 'aggregator'
FROM (
SELECT github_owner, github_repo
FROM skills WHERE is_blocked = false
GROUP BY github_owner, github_repo
HAVING COUNT(*) >= 50
) agg
WHERE s.github_owner = agg.github_owner
AND s.github_repo = agg.github_repo
AND s.is_blocked = false
AND s.skill_type IS NULL
`);
console.log(` Marked as aggregator: ${n(result.rowCount)} skills`);
}
// ─── Step 3: Classify remaining repos ───
async function step3_classifyRemaining() {
header(3, 'CLASSIFY REMAINING REPOS');
// 3a: repos with 10-49 skills, name contains marketplace/awesome/collection → aggregator
if (!DRY_RUN) {
const aggByName = await query(`
UPDATE skills s SET skill_type = 'aggregator'
FROM (
SELECT github_owner, github_repo
FROM skills WHERE is_blocked = false AND skill_type IS NULL
GROUP BY github_owner, github_repo
HAVING COUNT(*) >= 10
AND (github_repo ILIKE '%marketplace%'
OR github_repo ILIKE '%awesome%'
OR github_repo ILIKE '%collection%'
OR github_repo ILIKE '%registry%')
) agg
WHERE s.github_owner = agg.github_owner
AND s.github_repo = agg.github_repo
AND s.is_blocked = false
AND s.skill_type IS NULL
`);
console.log(` Aggregator by name (10+ & marketplace/awesome/registry): ${n(aggByName.rowCount)}`);
}
// 3b: repos with 3-49 skills → collection
if (!DRY_RUN) {
const collections = await query(`
UPDATE skills s SET skill_type = 'collection'
FROM (
SELECT github_owner, github_repo
FROM skills WHERE is_blocked = false AND skill_type IS NULL
GROUP BY github_owner, github_repo
HAVING COUNT(*) >= 3
) col
WHERE s.github_owner = col.github_owner
AND s.github_repo = col.github_repo
AND s.is_blocked = false
AND s.skill_type IS NULL
`);
console.log(` Collection (3+ skills in repo): ${n(collections.rowCount)}`);
}
// 3c: single/two-skill repos with project-bound name patterns → project-bound
if (!DRY_RUN) {
const projectBound = await query(`
UPDATE skills SET skill_type = 'project-bound'
WHERE is_blocked = false AND skill_type IS NULL
AND repo_skill_count <= 2
AND (name ILIKE '%my-%' OR name ILIKE '%my\\_%' ESCAPE '\\'
OR name ILIKE '%project%' OR name ILIKE '%team%' OR name ILIKE '%internal%'
OR name ILIKE '%.mdc' OR name ILIKE '%cursorrule%'
OR name ILIKE '%config%' OR name ILIKE '%setup%')
`);
console.log(` Project-bound (name pattern): ${n(projectBound.rowCount)}`);
}
// 3d: remaining → standalone
if (!DRY_RUN) {
const standalone = await query(`
UPDATE skills SET skill_type = 'standalone'
WHERE is_blocked = false AND skill_type IS NULL
`);
console.log(` Standalone (remaining): ${n(standalone.rowCount)}`);
}
// Summary
const summary = await query(`
SELECT skill_type, COUNT(*)::int AS count
FROM skills WHERE is_blocked = false
GROUP BY skill_type ORDER BY count DESC
`);
console.log('\n Classification summary:');
for (const r of summary.rows) {
console.log(` ${(r.skill_type || 'null').padEnd(15)} ${n(r.count)}`);
}
}
// ─── Step 4: Fill missing content_hash ───
async function step4_fillContentHash() {
header(4, 'FILL MISSING content_hash');
const countResult = await query(`
SELECT COUNT(*)::int AS total
FROM skills WHERE is_blocked = false AND content_hash IS NULL AND raw_content IS NOT NULL
`);
const missing = countResult.rows[0].total;
console.log(` Skills missing content_hash: ${n(missing)}`);
if (missing === 0) { console.log(' Nothing to do'); return; }
if (DRY_RUN) { console.log(' [DRY RUN] Would compute hash for these skills'); return; }
// Use md5 as a reliable hash function available in PostgreSQL
const result = await query(`
UPDATE skills SET content_hash = md5(raw_content)
WHERE is_blocked = false AND content_hash IS NULL AND raw_content IS NOT NULL
`);
console.log(` Computed content_hash (md5) for: ${n(result.rowCount)} skills`);
// Note: existing hashes use a JS numeric hash (analyzer.ts hashContent).
// We now use md5 for missing ones. For dedup purposes, same content → same hash
// regardless of algorithm, since we compare within hash groups.
// But mixed algorithms mean same content could have different hashes.
// Solution: re-hash ALL with md5 for consistency.
console.log(' Re-hashing ALL skills with md5 for consistency...');
const rehash = await query(`
UPDATE skills SET content_hash = md5(raw_content)
WHERE is_blocked = false AND raw_content IS NOT NULL
`);
console.log(` Re-hashed: ${n(rehash.rowCount)} skills`);
}
// ─── Step 5: Mark duplicates ───
async function step5_markDuplicates() {
header(5, 'MARK DUPLICATES BY content_hash');
// Find duplicate groups
const dupGroups = await query(`
SELECT content_hash, COUNT(*)::int AS copies
FROM skills
WHERE is_blocked = false AND content_hash IS NOT NULL AND is_duplicate = false
GROUP BY content_hash
HAVING COUNT(*) > 1
ORDER BY copies DESC
`);
const totalDupGroups = dupGroups.rowCount;
const totalDupSkills = dupGroups.rows.reduce((sum, r) => sum + r.copies, 0);
const removable = dupGroups.rows.reduce((sum, r) => sum + r.copies - 1, 0);
console.log(` Duplicate groups: ${n(totalDupGroups)}`);
console.log(` Total skills in dup groups: ${n(totalDupSkills)}`);
console.log(` Removable (keeping 1 per group): ${n(removable)}`);
console.log(` Top 5 by copies:`);
for (const r of dupGroups.rows.slice(0, 5)) {
console.log(` hash=${r.content_hash}: ${r.copies} copies`);
}
if (DRY_RUN) { console.log(' [DRY RUN] Would mark duplicates'); return; }
// For each duplicate group:
// - canonical = highest github_stars, then oldest created_at
// - rest = is_duplicate = true, canonical_skill_id = canonical.id
const markResult = await query(`
WITH ranked AS (
SELECT id, content_hash,
ROW_NUMBER() OVER (
PARTITION BY content_hash
ORDER BY github_stars DESC NULLS LAST,
created_at ASC
) AS rn
FROM skills
WHERE is_blocked = false AND content_hash IS NOT NULL AND is_duplicate = false
),
canonicals AS (
SELECT content_hash, id AS canonical_id
FROM ranked WHERE rn = 1
)
UPDATE skills s
SET is_duplicate = true,
canonical_skill_id = c.canonical_id
FROM ranked r
JOIN canonicals c ON r.content_hash = c.content_hash
WHERE s.id = r.id
AND r.rn > 1
`);
console.log(` Marked as duplicate: ${n(markResult.rowCount)} skills`);
// Verify
const verify = await query(`
SELECT
COUNT(*) FILTER (WHERE is_duplicate = false)::int AS unique_skills,
COUNT(*) FILTER (WHERE is_duplicate = true)::int AS duplicates
FROM skills WHERE is_blocked = false
`);
console.log(` After dedup: ${n(verify.rows[0].unique_skills)} unique, ${n(verify.rows[0].duplicates)} duplicates`);
}
// ─── Step 6: Detect fork marketplace repos ───
async function step6_detectForks() {
header(6, 'DETECT FORK MARKETPLACE REPOS');
// Known fork patterns: repos with same name across many owners
const forkPatterns = await query(`
SELECT github_repo, COUNT(DISTINCT github_owner)::int AS owners,
COUNT(*)::int AS total_skills
FROM skills
WHERE is_blocked = false
AND repo_skill_count >= 20
GROUP BY github_repo
HAVING COUNT(DISTINCT github_owner) >= 3
ORDER BY owners DESC
`);
console.log(` Repo names appearing in 3+ owners (with 20+ skills each):`);
for (const r of forkPatterns.rows) {
console.log(` ${r.github_repo}: ${r.owners} owners, ${n(r.total_skills)} skills`);
}
if (DRY_RUN) { console.log(' [DRY RUN] Would mark fork repo skills'); return; }
// For fork repos, all copies should be checked as duplicates
// The canonical is already handled by step 5 (content_hash dedup)
// Here we just ensure they're typed correctly as aggregator
if (forkPatterns.rows.length > 0) {
const repoNames = forkPatterns.rows.map(r => r.github_repo);
const placeholders = repoNames.map((_, i) => `$${i + 1}`).join(',');
const markForks = await query(`
UPDATE skills SET skill_type = 'aggregator'
WHERE is_blocked = false
AND github_repo IN (${placeholders})
AND repo_skill_count >= 20
AND (skill_type IS NULL OR skill_type != 'aggregator')
`, repoNames);
console.log(` Re-classified as aggregator (fork repos): ${n(markForks.rowCount)}`);
}
}
// ─── Step 7: Recalculate category counts ───
async function step7_categoryCounts() {
header(7, 'RECALCULATE CATEGORY COUNTS');
// Update skill_count on each category to only count browse-ready skills
const result = await query(`
UPDATE categories c
SET skill_count = sub.cnt
FROM (
SELECT sc.category_id, COUNT(*)::int AS cnt
FROM skill_categories sc
JOIN skills s ON sc.skill_id = s.id
WHERE s.is_blocked = false
AND s.is_duplicate = false
AND (s.skill_type IS NULL OR s.skill_type != 'aggregator')
GROUP BY sc.category_id
) sub
WHERE c.id = sub.category_id
AND c.skill_count IS DISTINCT FROM sub.cnt
`);
console.log(` Updated ${result.rowCount} category counts (browse-ready only)`);
// Also zero out categories that have no browse-ready skills
const zeroResult = await query(`
UPDATE categories c
SET skill_count = 0
WHERE c.skill_count > 0
AND NOT EXISTS (
SELECT 1 FROM skill_categories sc
JOIN skills s ON sc.skill_id = s.id
WHERE sc.category_id = c.id
AND s.is_blocked = false
AND s.is_duplicate = false
AND (s.skill_type IS NULL OR s.skill_type != 'aggregator')
)
`);
if (zeroResult.rowCount > 0) {
console.log(` Zeroed ${zeroResult.rowCount} categories with no browse-ready skills`);
}
// Show category counts
const cats = await query(`
SELECT name, skill_count FROM categories
WHERE id NOT LIKE 'parent-%'
ORDER BY skill_count DESC
LIMIT 10
`);
console.log(' Top 10 categories by browse-ready count:');
for (const r of cats.rows) {
console.log(` ${n(r.skill_count).padStart(6)} ${r.name}`);
}
}
// ─── Step 8: Summary ───
async function step8_summary() {
header(8, 'FINAL SUMMARY');
const summary = await query(`
SELECT
COUNT(*)::int AS total,
COUNT(*) FILTER (WHERE is_duplicate = false)::int AS unique_skills,
COUNT(*) FILTER (WHERE is_duplicate = true)::int AS duplicates,
COUNT(*) FILTER (WHERE skill_type = 'standalone' AND is_duplicate = false)::int AS standalone,
COUNT(*) FILTER (WHERE skill_type = 'collection' AND is_duplicate = false)::int AS collection,
COUNT(*) FILTER (WHERE skill_type = 'aggregator' AND is_duplicate = false)::int AS aggregator,
COUNT(*) FILTER (WHERE skill_type = 'project-bound' AND is_duplicate = false)::int AS project_bound,
COUNT(*) FILTER (WHERE skill_type IS NULL AND is_duplicate = false)::int AS unclassified
FROM skills WHERE is_blocked = false
`);
const s = summary.rows[0];
console.log(`
┌─────────────────────────────────────────────────────┐
│ CURATION SUMMARY │
├─────────────────────────────────────────────────────┤
│ Total skills: ${n(s.total).padStart(8)} │
│ Unique skills: ${n(s.unique_skills).padStart(8)} │
│ Duplicates: ${n(s.duplicates).padStart(8)} │
├─────────────────────────────────────────────────────┤
│ Unique by type: │
│ Standalone: ${n(s.standalone).padStart(8)} │
│ Collection: ${n(s.collection).padStart(8)} │
│ Aggregator: ${n(s.aggregator).padStart(8)} │
│ Project-bound: ${n(s.project_bound).padStart(8)} │
│ Unclassified: ${n(s.unclassified).padStart(8)} │
└─────────────────────────────────────────────────────┘
`);
// Interesting standalone skills
const topStandalone = await query(`
SELECT id, name, github_stars AS stars, COALESCE(download_count,0) AS dl,
LEFT(description, 70) AS description
FROM skills
WHERE is_blocked = false AND is_duplicate = false
AND skill_type = 'standalone'
ORDER BY github_stars DESC NULLS LAST
LIMIT 20
`);
console.log(' Top 20 standalone skills by stars:');
for (const r of topStandalone.rows) {
console.log(` [${n(r.stars)}★ ${n(r.dl)}↓] ${r.id}`);
console.log(` ${r.description}`);
}
// Browse-ready count (what users would see)
const browseReady = await query(`
SELECT COUNT(*)::int AS count
FROM skills
WHERE is_blocked = false
AND is_duplicate = false
AND skill_type IN ('standalone', 'collection')
`);
console.log(`\n Browse-ready skills (standalone + collection, unique): ${n(browseReady.rows[0].count)}`);
}
// ─── Main ───
async function main() {
const t0 = Date.now();
await connect();
if (DRY_RUN) console.log('\n *** DRY RUN MODE — no changes will be made ***\n');
const steps = [
step1_repoSkillCount,
step2_markAggregators,
step3_classifyRemaining,
step4_fillContentHash,
step5_markDuplicates,
step6_detectForks,
step7_categoryCounts,
step8_summary,
];
for (let i = 0; i < steps.length; i++) {
if (ONLY_STEP && ONLY_STEP !== i + 1) continue;
await steps[i]();
}
const dur = ((Date.now() - t0) / 1000).toFixed(1);
console.log(`\nCompleted in ${dur}s`);
await client.end().catch(() => {});
}
main().catch(async e => {
console.error('FAILED:', e.message || e);
await client?.end().catch(() => {});
process.exit(1);
});

214
scripts/curation/curate.sql
View File

@@ -1,214 +0,0 @@
-- ============================================================
-- Phase 2: Data Cleanup & Classification (Pure SQL version)
-- Run inside DB container:
-- psql -U postgres -d skillhub -f /tmp/curate.sql
--
-- Safe to run multiple times (idempotent).
-- ============================================================
\echo '======================================================================'
\echo ' STEP 1: COMPUTE repo_skill_count'
\echo '======================================================================'
UPDATE skills s SET repo_skill_count = sub.cnt
FROM (
SELECT github_owner, github_repo, COUNT(*)::int AS cnt
FROM skills WHERE is_blocked = false
GROUP BY github_owner, github_repo
) sub
WHERE s.github_owner = sub.github_owner
AND s.github_repo = sub.github_repo
AND s.is_blocked = false;
\echo ' Step 1 done. Checking counts:'
SELECT 'repo_skill_count set' AS step,
COUNT(*) FILTER (WHERE repo_skill_count IS NOT NULL) AS done,
COUNT(*) FILTER (WHERE repo_skill_count IS NULL) AS remaining
FROM skills WHERE is_blocked = false;
\echo ''
\echo '======================================================================'
\echo ' STEP 2: MARK AGGREGATORS (repos with 50+ skills)'
\echo '======================================================================'
UPDATE skills s SET skill_type = 'aggregator'
FROM (
SELECT github_owner, github_repo
FROM skills WHERE is_blocked = false
GROUP BY github_owner, github_repo
HAVING COUNT(*) >= 50
) agg
WHERE s.github_owner = agg.github_owner
AND s.github_repo = agg.github_repo
AND s.is_blocked = false
AND s.skill_type IS NULL;
\echo ' Step 2 done.'
\echo ''
\echo '======================================================================'
\echo ' STEP 3: CLASSIFY REMAINING REPOS'
\echo '======================================================================'
-- 3a: 10-49 skills + name contains marketplace/awesome/collection/registry → aggregator
UPDATE skills s SET skill_type = 'aggregator'
FROM (
SELECT github_owner, github_repo
FROM skills WHERE is_blocked = false AND skill_type IS NULL
GROUP BY github_owner, github_repo
HAVING COUNT(*) >= 10
AND (github_repo ILIKE '%marketplace%'
OR github_repo ILIKE '%awesome%'
OR github_repo ILIKE '%collection%'
OR github_repo ILIKE '%registry%')
) agg
WHERE s.github_owner = agg.github_owner
AND s.github_repo = agg.github_repo
AND s.is_blocked = false
AND s.skill_type IS NULL;
-- 3b: 3+ skills → collection
UPDATE skills s SET skill_type = 'collection'
FROM (
SELECT github_owner, github_repo
FROM skills WHERE is_blocked = false AND skill_type IS NULL
GROUP BY github_owner, github_repo
HAVING COUNT(*) >= 3
) col
WHERE s.github_owner = col.github_owner
AND s.github_repo = col.github_repo
AND s.is_blocked = false
AND s.skill_type IS NULL;
-- 3c: project-bound name patterns
UPDATE skills SET skill_type = 'project-bound'
WHERE is_blocked = false AND skill_type IS NULL
AND repo_skill_count <= 2
AND (name ILIKE '%my-%' OR name ILIKE '%my\_%' ESCAPE '\'
OR name ILIKE '%project%' OR name ILIKE '%team%' OR name ILIKE '%internal%'
OR name ILIKE '%.mdc' OR name ILIKE '%cursorrule%'
OR name ILIKE '%config%' OR name ILIKE '%setup%');
-- 3d: remaining → standalone
UPDATE skills SET skill_type = 'standalone'
WHERE is_blocked = false AND skill_type IS NULL;
\echo ' Step 3 done. Classification:'
SELECT skill_type, COUNT(*)::int AS count
FROM skills WHERE is_blocked = false
GROUP BY skill_type ORDER BY count DESC;
\echo ''
\echo '======================================================================'
\echo ' STEP 4: FILL content_hash WITH md5 (for consistency)'
\echo '======================================================================'
-- Re-hash ALL with md5 for consistent dedup
UPDATE skills SET content_hash = md5(raw_content)
WHERE is_blocked = false AND raw_content IS NOT NULL;
\echo ' Step 4 done.'
SELECT 'content_hash' AS step,
COUNT(*) FILTER (WHERE content_hash IS NOT NULL) AS has_hash,
COUNT(*) FILTER (WHERE content_hash IS NULL) AS no_hash
FROM skills WHERE is_blocked = false;
\echo ''
\echo '======================================================================'
\echo ' STEP 5: MARK DUPLICATES BY content_hash'
\echo '======================================================================'
-- First reset any previous duplicate marks (for idempotency)
UPDATE skills SET is_duplicate = false, canonical_skill_id = NULL
WHERE is_blocked = false AND is_duplicate = true;
-- Mark duplicates: keep the one with most stars (or oldest) as canonical
WITH ranked AS (
SELECT id, content_hash,
ROW_NUMBER() OVER (
PARTITION BY content_hash
ORDER BY github_stars DESC NULLS LAST,
created_at ASC
) AS rn
FROM skills
WHERE is_blocked = false AND content_hash IS NOT NULL
),
canonicals AS (
SELECT content_hash, id AS canonical_id
FROM ranked WHERE rn = 1
)
UPDATE skills s
SET is_duplicate = true,
canonical_skill_id = c.canonical_id
FROM ranked r
JOIN canonicals c ON r.content_hash = c.content_hash
WHERE s.id = r.id
AND r.rn > 1;
\echo ' Step 5 done.'
SELECT 'dedup' AS step,
COUNT(*) FILTER (WHERE is_duplicate = false) AS unique_skills,
COUNT(*) FILTER (WHERE is_duplicate = true) AS duplicates
FROM skills WHERE is_blocked = false;
\echo ''
\echo '======================================================================'
\echo ' STEP 6: DETECT FORK MARKETPLACE REPOS'
\echo '======================================================================'
-- Re-classify fork repos (same repo name in 3+ owners with 20+ skills) as aggregator
UPDATE skills SET skill_type = 'aggregator'
WHERE is_blocked = false
AND repo_skill_count >= 20
AND (skill_type IS NULL OR skill_type != 'aggregator')
AND github_repo IN (
SELECT github_repo
FROM skills
WHERE is_blocked = false AND repo_skill_count >= 20
GROUP BY github_repo
HAVING COUNT(DISTINCT github_owner) >= 3
);
\echo ' Step 6 done. Fork patterns:'
SELECT github_repo, COUNT(DISTINCT github_owner)::int AS owners,
COUNT(*)::int AS total_skills
FROM skills
WHERE is_blocked = false AND repo_skill_count >= 20
GROUP BY github_repo
HAVING COUNT(DISTINCT github_owner) >= 3
ORDER BY owners DESC
LIMIT 15;
\echo ''
\echo '======================================================================'
\echo ' STEP 7: FINAL SUMMARY'
\echo '======================================================================'
SELECT
COUNT(*)::int AS total,
COUNT(*) FILTER (WHERE is_duplicate = false)::int AS unique_skills,
COUNT(*) FILTER (WHERE is_duplicate = true)::int AS duplicates,
COUNT(*) FILTER (WHERE skill_type = 'standalone' AND is_duplicate = false)::int AS standalone,
COUNT(*) FILTER (WHERE skill_type = 'collection' AND is_duplicate = false)::int AS collection_type,
COUNT(*) FILTER (WHERE skill_type = 'aggregator' AND is_duplicate = false)::int AS aggregator,
COUNT(*) FILTER (WHERE skill_type = 'project-bound' AND is_duplicate = false)::int AS project_bound,
COUNT(*) FILTER (WHERE skill_type IN ('standalone','collection') AND is_duplicate = false)::int AS browse_ready
FROM skills WHERE is_blocked = false;
\echo ''
\echo ' Top 20 standalone skills by stars:'
SELECT id, github_stars AS stars, COALESCE(download_count,0) AS dl,
LEFT(description, 70) AS description
FROM skills
WHERE is_blocked = false AND is_duplicate = false AND skill_type = 'standalone'
ORDER BY github_stars DESC NULLS LAST
LIMIT 20;
\echo ''
\echo ' DONE!'

652
scripts/curation/explore.mjs
View File

@@ -1,652 +0,0 @@
#!/usr/bin/env node
/**
* Phase 1: Database Exploration — Single-Query Version
*
* Runs ALL analytics in ONE SQL query to handle unstable connections.
*
* Usage:
* PGSSLMODE=disable DATABASE_URL=postgres://... node scripts/curation/explore.mjs
*/
import { createRequire } from 'module';
import { writeFileSync } from 'fs';
import { resolve, dirname } from 'path';
import { fileURLToPath } from 'url';
const __dirname = dirname(fileURLToPath(import.meta.url));
const projectRoot = resolve(__dirname, '../..');
let pg;
const tryPaths = [
'/tmp/package.json',
process.env.APPDATA ? resolve(process.env.APPDATA, '..', 'Local', 'Temp', 'package.json') : null,
resolve(projectRoot, 'package.json'),
].filter(Boolean);
for (const p of tryPaths) {
try { pg = createRequire(p)('pg'); break; } catch {}
}
if (!pg) { console.error('pg not found. Run: cd /tmp && npm install pg'); process.exit(1); }
const DATABASE_URL = process.env.DATABASE_URL || 'postgresql://postgres:postgres@localhost:5432/skillhub';
process.on('uncaughtException', e => {
if (e.code === 'ECONNRESET') return;
console.error('Fatal:', e); process.exit(1);
});
// ─── Mega Query ──────────────────────────────────────────
// All analytics combined into one JSON result
const MEGA_SQL = `
WITH active AS (
SELECT * FROM skills WHERE is_blocked = false
),
-- 1. Overall
overall AS (
SELECT json_build_object(
'total', (SELECT COUNT(*)::int FROM skills),
'active', (SELECT COUNT(*)::int FROM active),
'blocked', (SELECT COUNT(*) FILTER (WHERE is_blocked)::int FROM skills),
'owners', (SELECT COUNT(DISTINCT github_owner)::int FROM active),
'repos', (SELECT COUNT(DISTINCT github_owner||'/'||github_repo)::int FROM active),
'skillmd', (SELECT COUNT(*) FILTER (WHERE source_format='skill.md')::int FROM active),
'other_fmt', (SELECT COUNT(*) FILTER (WHERE source_format!='skill.md')::int FROM active),
'downloads', (SELECT COALESCE(SUM(download_count),0)::bigint FROM active),
'views', (SELECT COALESCE(SUM(view_count),0)::bigint FROM active),
'rated', (SELECT COUNT(*) FILTER (WHERE rating_count>0)::int FROM active),
'first', (SELECT MIN(created_at)::text FROM active),
'last', (SELECT MAX(created_at)::text FROM active)
) AS data
),
-- 1b. Source formats
source_formats AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT COALESCE(source_format,'null') AS format, COUNT(*)::int AS count,
COUNT(*) FILTER (WHERE is_blocked=false)::int AS active
FROM skills GROUP BY source_format ORDER BY count DESC
) t
),
-- 2. Stars distribution
stars_dist AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT CASE
WHEN github_stars IS NULL OR github_stars=0 THEN '0'
WHEN github_stars BETWEEN 1 AND 5 THEN '1-5'
WHEN github_stars BETWEEN 6 AND 10 THEN '6-10'
WHEN github_stars BETWEEN 11 AND 50 THEN '11-50'
WHEN github_stars BETWEEN 51 AND 100 THEN '51-100'
WHEN github_stars BETWEEN 101 AND 500 THEN '101-500'
WHEN github_stars BETWEEN 501 AND 1000 THEN '501-1K'
WHEN github_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
WHEN github_stars BETWEEN 5001 AND 10000 THEN '5K-10K'
WHEN github_stars BETWEEN 10001 AND 50000 THEN '10K-50K'
ELSE '50K+'
END AS stars, COUNT(*)::int AS skills,
COUNT(DISTINCT github_owner)::int AS owners,
COUNT(DISTINCT github_owner||'/'||github_repo)::int AS repos
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(github_stars,0))
) t
),
-- 3. Downloads
dl_dist AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT CASE
WHEN COALESCE(download_count,0)=0 THEN '0'
WHEN download_count BETWEEN 1 AND 5 THEN '1-5'
WHEN download_count BETWEEN 6 AND 50 THEN '6-50'
WHEN download_count BETWEEN 51 AND 500 THEN '51-500'
ELSE '500+'
END AS range, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(download_count,0))
) t
),
-- 3b. Views
vw_dist AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT CASE
WHEN COALESCE(view_count,0)=0 THEN '0'
WHEN view_count BETWEEN 1 AND 10 THEN '1-10'
WHEN view_count BETWEEN 11 AND 100 THEN '11-100'
WHEN view_count BETWEEN 101 AND 1000 THEN '101-1K'
ELSE '1K+'
END AS range, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(view_count,0))
) t
),
-- 3c. Top downloaded
top_dl AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT id, name, download_count AS dl, view_count AS views,
github_stars AS stars, github_owner AS owner
FROM active WHERE COALESCE(download_count,0)>0
ORDER BY download_count DESC LIMIT 20
) t
),
-- 3d. Top viewed
top_vw AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT id, name, view_count AS views, download_count AS dl,
github_stars AS stars, github_owner AS owner
FROM active WHERE COALESCE(view_count,0)>0
ORDER BY view_count DESC LIMIT 20
) t
),
-- 4. Security
security AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT COALESCE(security_status,'null') AS status, COUNT(*)::int AS count
FROM active GROUP BY security_status ORDER BY count DESC
) t
),
-- 5. Content stats
content_stats AS (
SELECT json_build_object(
'has_content', COUNT(*) FILTER (WHERE raw_content IS NOT NULL)::int,
'no_content', COUNT(*) FILTER (WHERE raw_content IS NULL)::int,
'avg_len', ROUND(AVG(LENGTH(raw_content)) FILTER (WHERE raw_content IS NOT NULL))::int,
'max_len', MAX(LENGTH(raw_content))::int,
'median_len', PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY LENGTH(raw_content))
FILTER (WHERE raw_content IS NOT NULL)::int,
'avg_desc', ROUND(AVG(LENGTH(description)))::int,
'good_desc', COUNT(*) FILTER (WHERE LENGTH(description)>100)::int,
'short_desc', COUNT(*) FILTER (WHERE LENGTH(description)<=20)::int,
'has_ver', COUNT(*) FILTER (WHERE version IS NOT NULL)::int,
'has_lic', COUNT(*) FILTER (WHERE license IS NOT NULL)::int,
'has_auth', COUNT(*) FILTER (WHERE author IS NOT NULL)::int,
'has_hp', COUNT(*) FILTER (WHERE homepage IS NOT NULL)::int
) AS data FROM active
),
-- 5b. Content length dist
content_len AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT CASE
WHEN raw_content IS NULL THEN 'null'
WHEN LENGTH(raw_content)=0 THEN 'empty'
WHEN LENGTH(raw_content)<200 THEN '<200'
WHEN LENGTH(raw_content)<500 THEN '200-500'
WHEN LENGTH(raw_content)<1000 THEN '500-1K'
WHEN LENGTH(raw_content)<3000 THEN '1K-3K'
WHEN LENGTH(raw_content)<5000 THEN '3K-5K'
WHEN LENGTH(raw_content)<10000 THEN '5K-10K'
WHEN LENGTH(raw_content)<50000 THEN '10K-50K'
ELSE '50K+'
END AS range, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(LENGTH(raw_content),-1))
) t
),
-- 6. Triggers
trigger_stats AS (
SELECT json_build_object(
'has_triggers', COUNT(*) FILTER (WHERE triggers IS NOT NULL)::int,
'no_triggers', COUNT(*) FILTER (WHERE triggers IS NULL)::int,
'file_pats', COUNT(*) FILTER (WHERE triggers->>'filePatterns' IS NOT NULL
AND triggers->>'filePatterns'!='[]' AND triggers->>'filePatterns'!='null')::int,
'keywords', COUNT(*) FILTER (WHERE triggers->>'keywords' IS NOT NULL
AND triggers->>'keywords'!='[]' AND triggers->>'keywords'!='null')::int,
'languages', COUNT(*) FILTER (WHERE triggers->>'languages' IS NOT NULL
AND triggers->>'languages'!='[]' AND triggers->>'languages'!='null')::int,
'has_compat', COUNT(*) FILTER (WHERE compatibility IS NOT NULL)::int,
'platforms', COUNT(*) FILTER (WHERE compatibility->>'platforms' IS NOT NULL
AND compatibility->>'platforms'!='[]' AND compatibility->>'platforms'!='null')::int
) AS data FROM active
),
-- 7. Freshness
freshness AS (
SELECT json_build_object(
'created', (SELECT json_agg(row_to_json(t)) FROM (
SELECT CASE
WHEN created_at>NOW()-INTERVAL '7 days' THEN 'Last 7d'
WHEN created_at>NOW()-INTERVAL '30 days' THEN 'Last 30d'
WHEN created_at>NOW()-INTERVAL '90 days' THEN 'Last 90d'
WHEN created_at>NOW()-INTERVAL '180 days' THEN 'Last 180d'
ELSE 'Older'
END AS range, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY MIN(NOW()-created_at)
) t),
'updated', (SELECT json_agg(row_to_json(t)) FROM (
SELECT CASE
WHEN updated_at>NOW()-INTERVAL '7 days' THEN 'Last 7d'
WHEN updated_at>NOW()-INTERVAL '30 days' THEN 'Last 30d'
WHEN updated_at>NOW()-INTERVAL '90 days' THEN 'Last 90d'
WHEN updated_at>NOW()-INTERVAL '180 days' THEN 'Last 180d'
ELSE 'Older'
END AS range, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY MIN(NOW()-updated_at)
) t),
'last_dl', (SELECT json_agg(row_to_json(t)) FROM (
SELECT CASE
WHEN last_downloaded_at IS NULL THEN 'Never'
WHEN last_downloaded_at>NOW()-INTERVAL '30 days' THEN 'Last 30d'
WHEN last_downloaded_at>NOW()-INTERVAL '90 days' THEN 'Last 90d'
ELSE 'Older'
END AS range, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(last_downloaded_at,'1970-01-01'::timestamp))
) t)
) AS data
),
-- 8. Top owners
top_owners_count AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT github_owner AS owner, COUNT(*)::int AS skills,
COUNT(DISTINCT github_repo)::int AS repos,
MAX(github_stars)::int AS max_stars,
COALESCE(SUM(download_count),0)::int AS dl
FROM active GROUP BY github_owner ORDER BY skills DESC LIMIT 30
) t
),
top_owners_stars AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT github_owner AS owner, MAX(github_stars)::int AS max_stars,
COUNT(*)::int AS skills, COUNT(DISTINCT github_repo)::int AS repos
FROM active GROUP BY github_owner ORDER BY max_stars DESC LIMIT 20
) t
),
owner_concentration AS (
SELECT json_build_object(
'top1', SUM(cnt) FILTER (WHERE rn<=1)::int,
'top5', SUM(cnt) FILTER (WHERE rn<=5)::int,
'top10', SUM(cnt) FILTER (WHERE rn<=10)::int,
'top20', SUM(cnt) FILTER (WHERE rn<=20)::int,
'top50', SUM(cnt) FILTER (WHERE rn<=50)::int,
'total', SUM(cnt)::int,
'owners', COUNT(*)::int
) AS data FROM (
SELECT github_owner, COUNT(*)::int AS cnt,
ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn
FROM active GROUP BY github_owner
) x
),
-- 9. Multi-skill repos
aggregators AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT github_owner||'/'||github_repo AS repo, COUNT(*)::int AS skills,
MAX(github_stars)::int AS stars
FROM active GROUP BY github_owner, github_repo HAVING COUNT(*)>=20
ORDER BY skills DESC LIMIT 30
) t
),
collections AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT github_owner||'/'||github_repo AS repo, COUNT(*)::int AS skills,
MAX(github_stars)::int AS stars
FROM active GROUP BY github_owner, github_repo HAVING COUNT(*) BETWEEN 3 AND 19
ORDER BY skills DESC LIMIT 30
) t
),
skills_per_repo AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
WITH rc AS (SELECT COUNT(*)::int AS cnt FROM active GROUP BY github_owner, github_repo)
SELECT CASE
WHEN cnt=1 THEN '1' WHEN cnt=2 THEN '2'
WHEN cnt BETWEEN 3 AND 5 THEN '3-5' WHEN cnt BETWEEN 6 AND 10 THEN '6-10'
WHEN cnt BETWEEN 11 AND 20 THEN '11-20' WHEN cnt BETWEEN 21 AND 50 THEN '21-50'
WHEN cnt BETWEEN 51 AND 100 THEN '51-100' ELSE '100+'
END AS per_repo, COUNT(*)::int AS repos, SUM(cnt)::int AS total_skills
FROM rc GROUP BY 1 ORDER BY MIN(cnt)
) t
),
-- 10. Categories
categories_dist AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT c.name AS category, c.skill_count::int AS skills
FROM categories c WHERE c.id NOT LIKE 'parent-%' ORDER BY c.skill_count DESC
) t
),
uncategorized AS (
SELECT COUNT(*)::int AS data FROM active s
WHERE NOT EXISTS (SELECT 1 FROM skill_categories sc WHERE sc.skill_id=s.id)
),
-- 11. Flags
flags AS (
SELECT json_build_object(
'verified', COUNT(*) FILTER (WHERE is_verified)::int,
'featured', COUNT(*) FILTER (WHERE is_featured)::int,
'user_rated', COUNT(*) FILTER (WHERE rating_count>0)::int,
'avg_rating', ROUND(AVG(rating) FILTER (WHERE rating IS NOT NULL),2)::numeric,
'max_ratings', MAX(rating_count)::int
) AS data FROM active
),
-- 12. Cross-table
cross_stats AS (
SELECT json_build_object(
'users', (SELECT COUNT(*)::int FROM users),
'ratings', (SELECT COUNT(*)::int FROM ratings),
'installs', (SELECT COUNT(*)::int FROM installations),
'favorites', (SELECT COUNT(*)::int FROM favorites),
'disc_repos', (SELECT COUNT(*)::int FROM discovered_repos),
'disc_with_skills', (SELECT COUNT(*) FILTER (WHERE has_skill_md)::int FROM discovered_repos),
'removals', (SELECT COUNT(*)::int FROM removal_requests),
'adds', (SELECT COUNT(*)::int FROM add_requests),
'subs', (SELECT COUNT(*) FILTER (WHERE unsubscribed_at IS NULL)::int FROM email_subscriptions)
) AS data
),
install_platform AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT platform, COUNT(*)::int AS count FROM installations GROUP BY platform ORDER BY count DESC
) t
),
-- 13. Usability signals
usability AS (
SELECT json_build_object(
'strong', COUNT(*) FILTER (WHERE LENGTH(description)>50 AND raw_content IS NOT NULL
AND LENGTH(raw_content)>500 AND security_status='pass')::int,
'file_triggers', COUNT(*) FILTER (WHERE triggers IS NOT NULL AND triggers!='{}'
AND triggers->>'filePatterns' IS NOT NULL AND triggers->>'filePatterns'!='[]')::int,
'downloaded', COUNT(*) FILTER (WHERE COALESCE(download_count,0)>0)::int,
'high_q', COUNT(*) FILTER (WHERE github_stars>=10 AND LENGTH(description)>50
AND raw_content IS NOT NULL AND security_status='pass')::int,
'premium', COUNT(*) FILTER (WHERE github_stars>=100 AND COALESCE(download_count,0)>0
AND security_status='pass')::int,
'skillmd_q', COUNT(*) FILTER (WHERE source_format='skill.md' AND LENGTH(description)>50
AND raw_content IS NOT NULL AND LENGTH(raw_content)>300 AND security_status='pass')::int
) AS data FROM active
),
-- 14. Standalone vs project-bound
repo_types AS (
SELECT json_build_object(
'single', COUNT(*) FILTER (WHERE cnt=1)::int,
'two', COUNT(*) FILTER (WHERE cnt=2)::int,
'multi', COUNT(*) FILTER (WHERE cnt>=3)::int,
'total', COUNT(*)::int
) AS data FROM (
SELECT COUNT(*)::int AS cnt FROM active GROUP BY github_owner, github_repo
) x
),
name_patterns AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT CASE
WHEN name ILIKE '%rule%' OR name ILIKE '%config%' OR name ILIKE '%setup%' THEN 'rules/config/setup'
WHEN name ILIKE '%my-%' OR name ILIKE '%my_%' THEN 'my-prefix'
WHEN name ILIKE '%cursor%' OR name ILIKE '%claude%' OR name ILIKE '%copilot%' THEN 'tool-specific'
WHEN name ILIKE '%project%' OR name ILIKE '%team%' OR name ILIKE '%internal%' THEN 'project/team'
WHEN name ILIKE '%.mdc' OR name ILIKE '%cursorrule%' THEN 'cursor-rules'
ELSE 'generic'
END AS pattern, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY skills DESC
) t
),
-- 15. Samples
top_by_stars AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT id, name, github_stars AS stars, COALESCE(download_count,0) AS dl,
security_status AS sec, source_format AS fmt, LEFT(description,80) AS description
FROM active ORDER BY github_stars DESC LIMIT 15
) t
),
top_by_dl AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT id, name, download_count AS dl, view_count AS views,
github_stars AS stars, LEFT(description,80) AS description
FROM active WHERE COALESCE(download_count,0)>0
ORDER BY download_count DESC LIMIT 15
) t
),
-- 16. Discovered repos
discovered AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT discovered_via AS source, COUNT(*)::int AS total,
COUNT(*) FILTER (WHERE has_skill_md)::int AS with_skills,
COUNT(*) FILTER (WHERE last_scanned IS NOT NULL)::int AS scanned
FROM discovered_repos GROUP BY discovered_via ORDER BY total DESC
) t
),
-- 18. Cache
cache_stats AS (
SELECT json_build_object(
'cached', COUNT(*) FILTER (WHERE cached_files IS NOT NULL)::int,
'not_cached', COUNT(*) FILTER (WHERE cached_files IS NULL)::int
) AS data FROM active
),
-- 19. Branches
branches AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT COALESCE(branch,'null') AS branch, COUNT(*)::int AS count
FROM active GROUP BY branch ORDER BY count DESC LIMIT 10
) t
),
-- 20. Duplicates
exact_dupes AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT name, LEFT(description,60) AS desc, COUNT(*)::int AS copies,
STRING_AGG(DISTINCT github_owner,', ') AS owners
FROM active WHERE description IS NOT NULL AND LENGTH(description)>10
GROUP BY name, description HAVING COUNT(*)>1
ORDER BY copies DESC LIMIT 15
) t
),
hash_dupes AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT content_hash, COUNT(*)::int AS copies,
MIN(name) AS sample, STRING_AGG(DISTINCT github_owner,', ') AS owners
FROM active WHERE content_hash IS NOT NULL
GROUP BY content_hash HAVING COUNT(*)>2
ORDER BY copies DESC LIMIT 15
) t
)
-- Final: combine everything into one JSON
SELECT json_build_object(
'overall', (SELECT data FROM overall),
'sourceFormats', (SELECT data FROM source_formats),
'starsDist', (SELECT data FROM stars_dist),
'dlDist', (SELECT data FROM dl_dist),
'vwDist', (SELECT data FROM vw_dist),
'topDL', (SELECT data FROM top_dl),
'topVW', (SELECT data FROM top_vw),
'security', (SELECT data FROM security),
'contentStats', (SELECT data FROM content_stats),
'contentLen', (SELECT data FROM content_len),
'triggerStats', (SELECT data FROM trigger_stats),
'freshness', (SELECT data FROM freshness),
'topOwnersCount', (SELECT data FROM top_owners_count),
'topOwnersStars', (SELECT data FROM top_owners_stars),
'ownerConcentration', (SELECT data FROM owner_concentration),
'aggregators', (SELECT data FROM aggregators),
'collections', (SELECT data FROM collections),
'skillsPerRepo', (SELECT data FROM skills_per_repo),
'categories', (SELECT data FROM categories_dist),
'uncategorized', (SELECT data FROM uncategorized),
'flags', (SELECT data FROM flags),
'crossStats', (SELECT data FROM cross_stats),
'installPlatform', (SELECT data FROM install_platform),
'usability', (SELECT data FROM usability),
'repoTypes', (SELECT data FROM repo_types),
'namePatterns', (SELECT data FROM name_patterns),
'topByStars', (SELECT data FROM top_by_stars),
'topByDL', (SELECT data FROM top_by_dl),
'discovered', (SELECT data FROM discovered),
'cacheStats', (SELECT data FROM cache_stats),
'branches', (SELECT data FROM branches),
'exactDupes', (SELECT data FROM exact_dupes),
'hashDupes', (SELECT data FROM hash_dupes)
) AS report;
`;
// ─── Helpers ─────────────────────────────────────────────
function header(title) {
console.log(`\n${'='.repeat(70)}\n ${title}\n${'='.repeat(70)}`);
}
function sub(title) {
console.log(`\n -- ${title} ${'─'.repeat(Math.max(0, 58 - title.length))}`);
}
function tbl(rows, max = 30) {
if (!rows || rows.length === 0) { console.log(' (no data)'); return; }
const display = rows.slice(0, max);
const keys = Object.keys(display[0]);
const w = keys.map(k => Math.max(k.length, ...display.map(r => String(r[k] ?? '').length)));
console.log(' ' + keys.map((k, i) => k.padEnd(w[i])).join(' '));
console.log(' ' + w.map(x => '-'.repeat(x)).join(' '));
for (const row of display)
console.log(' ' + keys.map((k, i) => String(row[k] ?? '').padEnd(w[i])).join(' '));
if (rows.length > max) console.log(` ... and ${rows.length - max} more`);
}
const n = v => Number(v ?? 0).toLocaleString('en-US');
const p = (a, b) => (Number(b ?? 1) === 0 ? '0.0%' : (Number(a ?? 0) / Number(b) * 100).toFixed(1) + '%');
// ─── Run ─────────────────────────────────────────────────
async function run() {
const t0 = Date.now();
console.log('Connecting and running mega-query (this may take 30-60s)...');
const c = new pg.Client({ connectionString: DATABASE_URL, ssl: false,
connectionTimeoutMillis: 15000, query_timeout: 120000,
keepAlive: true, keepAliveInitialDelayMillis: 1000 });
await c.connect();
const result = await c.query(MEGA_SQL);
await c.end().catch(() => {});
const r = result.rows[0].report;
const dur = ((Date.now() - t0) / 1000).toFixed(1);
console.log(`Query completed in ${dur}s\n`);
// ── Display ──
const total = r.overall.active;
header('1. OVERALL STATISTICS');
const ov = r.overall;
console.log(` Total skills: ${n(ov.total)}`);
console.log(` Active (not blocked): ${n(ov.active)}`);
console.log(` Blocked: ${n(ov.blocked)}`);
console.log(` Unique owners: ${n(ov.owners)}`);
console.log(` Unique repos: ${n(ov.repos)}`);
console.log(` SKILL.md format: ${n(ov.skillmd)} (${p(ov.skillmd, ov.total)})`);
console.log(` Other formats: ${n(ov.other_fmt)} (${p(ov.other_fmt, ov.total)})`);
console.log(` Total downloads: ${n(ov.downloads)}`);
console.log(` Total views: ${n(ov.views)}`);
console.log(` Rated skills: ${n(ov.rated)}`);
console.log(` Date range: ${ov.first} -> ${ov.last}`);
sub('Source Formats');
tbl(r.sourceFormats);
header('2. GITHUB STARS');
tbl(r.starsDist);
header('3. ENGAGEMENT');
sub('Downloads'); tbl(r.dlDist);
sub('Views'); tbl(r.vwDist);
sub('Top 20 Downloaded'); tbl(r.topDL);
sub('Top 20 Viewed'); tbl(r.topVW);
header('4. SECURITY');
tbl(r.security);
header('5. CONTENT');
const cs = r.contentStats;
console.log(` Has content: ${n(cs.has_content)} (${p(cs.has_content, total)})`);
console.log(` No content: ${n(cs.no_content)} (${p(cs.no_content, total)})`);
console.log(` Avg length: ${n(cs.avg_len)} | Median: ${n(cs.median_len)} | Max: ${n(cs.max_len)}`);
console.log(` Avg desc: ${n(cs.avg_desc)} | Good(>100): ${n(cs.good_desc)} | Short(<=20): ${n(cs.short_desc)}`);
console.log(` Version: ${n(cs.has_ver)} | License: ${n(cs.has_lic)} | Author: ${n(cs.has_auth)} | HP: ${n(cs.has_hp)}`);
sub('Content Length Dist');
tbl(r.contentLen);
header('6. TRIGGERS');
const tr = r.triggerStats;
console.log(` Has triggers: ${n(tr.has_triggers)} (${p(tr.has_triggers, total)})`);
console.log(` filePatterns: ${n(tr.file_pats)} | keywords: ${n(tr.keywords)} | languages: ${n(tr.languages)}`);
console.log(` Compatibility: ${n(tr.has_compat)} | platforms: ${n(tr.platforms)}`);
header('7. FRESHNESS');
sub('Created'); tbl(r.freshness.created);
sub('Updated'); tbl(r.freshness.updated);
sub('Last DL'); tbl(r.freshness.last_dl);
header('8. TOP OWNERS');
sub('By Skill Count'); tbl(r.topOwnersCount);
sub('By Stars'); tbl(r.topOwnersStars);
sub('Concentration');
const oc = r.ownerConcentration;
console.log(` Top 1: ${n(oc.top1)} (${p(oc.top1, oc.total)}) | Top 5: ${n(oc.top5)} (${p(oc.top5, oc.total)}) | Top 10: ${n(oc.top10)} (${p(oc.top10, oc.total)})`);
console.log(` Top 20: ${n(oc.top20)} (${p(oc.top20, oc.total)}) | Top 50: ${n(oc.top50)} (${p(oc.top50, oc.total)}) | Total owners: ${n(oc.owners)}`);
header('9. MULTI-SKILL REPOS');
sub('Aggregators (20+)'); tbl(r.aggregators);
sub('Collections (3-19)'); tbl(r.collections);
sub('Skills-per-Repo'); tbl(r.skillsPerRepo);
header('10. CATEGORIES');
tbl(r.categories);
console.log(`\n Uncategorized: ${n(r.uncategorized)}`);
header('11. FLAGS');
const fl = r.flags;
console.log(` Verified: ${n(fl.verified)} | Featured: ${n(fl.featured)} | Rated: ${n(fl.user_rated)} | Avg: ${fl.avg_rating ?? 'N/A'}`);
header('12. CROSS-TABLE');
const xt = r.crossStats;
console.log(` Users: ${n(xt.users)} | Ratings: ${n(xt.ratings)} | Installs: ${n(xt.installs)} | Favs: ${n(xt.favorites)}`);
console.log(` Discovered: ${n(xt.disc_repos)} (${n(xt.disc_with_skills)} with skills) | Subs: ${n(xt.subs)}`);
sub('Install Platform'); tbl(r.installPlatform);
header('13. USABILITY SIGNALS');
const us = r.usability;
console.log(` Strong standalone: ${n(us.strong)} (desc>50 + content>500 + sec=pass)`);
console.log(` File triggers: ${n(us.file_triggers)}`);
console.log(` Ever downloaded: ${n(us.downloaded)}`);
console.log(` High quality: ${n(us.high_q)} (stars>=10 + desc + sec)`);
console.log(` Premium: ${n(us.premium)} (stars>=100 + dl>0 + sec)`);
console.log(` SKILL.md quality: ${n(us.skillmd_q)}`);
header('14. STANDALONE vs PROJECT-BOUND');
const sm = r.repoTypes;
console.log(` Single: ${n(sm.single)} (${p(sm.single,sm.total)}) | Two: ${n(sm.two)} | Multi: ${n(sm.multi)} | Total repos: ${n(sm.total)}`);
sub('Name Patterns'); tbl(r.namePatterns);
header('15. SAMPLES');
sub('Top by Stars'); tbl(r.topByStars);
sub('Top by DL'); tbl(r.topByDL);
header('16. DISCOVERED REPOS');
tbl(r.discovered);
header('17. CACHE');
const cf = r.cacheStats;
console.log(` Cached: ${n(cf.cached)} (${p(cf.cached,total)}) | Not: ${n(cf.not_cached)}`);
header('19. BRANCHES');
tbl(r.branches);
header('20. DUPLICATES');
sub('Same Name+Desc'); tbl(r.exactDupes);
sub('Same Hash (3+)'); tbl(r.hashDupes);
header(`COMPLETE (${dur}s)`);
// Save JSON
r.generatedAt = new Date().toISOString();
r.durationSeconds = parseFloat(dur);
const reportPath = resolve(__dirname, 'explore-report.json');
writeFileSync(reportPath, JSON.stringify(r, null, 2), 'utf-8');
console.log(`\n Report saved: ${reportPath}`);
}
run().catch(async e => {
console.error('FAILED:', e.message || e);
process.exit(1);
});

258
scripts/curation/explore.sql
View File

@@ -1,258 +0,0 @@
-- Phase 1: Database Exploration - Run inside container:
-- psql -U postgres skillhub -t -A -f /tmp/explore.sql > /tmp/report.json
-- Or paste this SQL directly in psql
WITH active AS (SELECT * FROM skills WHERE is_blocked = false),
overall AS (
SELECT json_build_object(
'total', (SELECT COUNT(*)::int FROM skills),
'active', (SELECT COUNT(*)::int FROM active),
'blocked', (SELECT COUNT(*) FILTER (WHERE is_blocked)::int FROM skills),
'owners', (SELECT COUNT(DISTINCT github_owner)::int FROM active),
'repos', (SELECT COUNT(DISTINCT github_owner||'/'||github_repo)::int FROM active),
'skillmd', (SELECT COUNT(*) FILTER (WHERE source_format='skill.md')::int FROM active),
'other_fmt', (SELECT COUNT(*) FILTER (WHERE source_format!='skill.md')::int FROM active),
'downloads', (SELECT COALESCE(SUM(download_count),0)::bigint FROM active),
'views', (SELECT COALESCE(SUM(view_count),0)::bigint FROM active),
'rated', (SELECT COUNT(*) FILTER (WHERE rating_count>0)::int FROM active),
'first', (SELECT MIN(created_at)::text FROM active),
'last', (SELECT MAX(created_at)::text FROM active)
) AS data
),
source_formats AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT COALESCE(source_format,'null') AS format, COUNT(*)::int AS count
FROM skills GROUP BY source_format ORDER BY count DESC
) t
),
stars_dist AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT CASE
WHEN github_stars IS NULL OR github_stars=0 THEN '0'
WHEN github_stars BETWEEN 1 AND 10 THEN '1-10'
WHEN github_stars BETWEEN 11 AND 50 THEN '11-50'
WHEN github_stars BETWEEN 51 AND 100 THEN '51-100'
WHEN github_stars BETWEEN 101 AND 500 THEN '101-500'
WHEN github_stars BETWEEN 501 AND 1000 THEN '501-1K'
WHEN github_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
WHEN github_stars BETWEEN 5001 AND 50000 THEN '5K-50K'
ELSE '50K+'
END AS stars, COUNT(*)::int AS skills,
COUNT(DISTINCT github_owner)::int AS owners
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(github_stars,0))
) t
),
dl_dist AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT CASE
WHEN COALESCE(download_count,0)=0 THEN '0'
WHEN download_count BETWEEN 1 AND 10 THEN '1-10'
WHEN download_count BETWEEN 11 AND 100 THEN '11-100'
ELSE '100+'
END AS range, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(download_count,0))
) t
),
top_dl AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT id, name, download_count AS dl, github_stars AS stars, github_owner AS owner
FROM active WHERE COALESCE(download_count,0)>0 ORDER BY download_count DESC LIMIT 20
) t
),
top_vw AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT id, name, view_count AS views, github_stars AS stars, github_owner AS owner
FROM active WHERE COALESCE(view_count,0)>0 ORDER BY view_count DESC LIMIT 20
) t
),
security AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT COALESCE(security_status,'null') AS status, COUNT(*)::int AS count
FROM active GROUP BY security_status ORDER BY count DESC
) t
),
content_stats AS (
SELECT json_build_object(
'has_content', COUNT(*) FILTER (WHERE raw_content IS NOT NULL)::int,
'no_content', COUNT(*) FILTER (WHERE raw_content IS NULL)::int,
'avg_len', ROUND(AVG(LENGTH(raw_content)) FILTER (WHERE raw_content IS NOT NULL))::int,
'max_len', MAX(LENGTH(raw_content))::int,
'median_len', PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY LENGTH(raw_content)) FILTER (WHERE raw_content IS NOT NULL)::int,
'avg_desc', ROUND(AVG(LENGTH(description)))::int,
'good_desc', COUNT(*) FILTER (WHERE LENGTH(description)>100)::int,
'short_desc', COUNT(*) FILTER (WHERE LENGTH(description)<=20)::int,
'has_ver', COUNT(*) FILTER (WHERE version IS NOT NULL)::int,
'has_lic', COUNT(*) FILTER (WHERE license IS NOT NULL)::int,
'has_auth', COUNT(*) FILTER (WHERE author IS NOT NULL)::int
) AS data FROM active
),
content_len AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT CASE
WHEN raw_content IS NULL THEN 'null'
WHEN LENGTH(raw_content)<200 THEN '<200'
WHEN LENGTH(raw_content)<1000 THEN '200-1K'
WHEN LENGTH(raw_content)<5000 THEN '1K-5K'
WHEN LENGTH(raw_content)<10000 THEN '5K-10K'
WHEN LENGTH(raw_content)<50000 THEN '10K-50K'
ELSE '50K+'
END AS range, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(LENGTH(raw_content),-1))
) t
),
trigger_stats AS (
SELECT json_build_object(
'has_triggers', COUNT(*) FILTER (WHERE triggers IS NOT NULL)::int,
'no_triggers', COUNT(*) FILTER (WHERE triggers IS NULL)::int,
'file_pats', COUNT(*) FILTER (WHERE triggers->>'filePatterns' IS NOT NULL AND triggers->>'filePatterns'!='[]')::int,
'keywords', COUNT(*) FILTER (WHERE triggers->>'keywords' IS NOT NULL AND triggers->>'keywords'!='[]')::int,
'platforms', COUNT(*) FILTER (WHERE compatibility->>'platforms' IS NOT NULL AND compatibility->>'platforms'!='[]')::int
) AS data FROM active
),
freshness AS (
SELECT json_build_object(
'created', (SELECT json_agg(row_to_json(t)) FROM (
SELECT CASE WHEN created_at>NOW()-INTERVAL '7d' THEN '7d' WHEN created_at>NOW()-INTERVAL '30d' THEN '30d'
WHEN created_at>NOW()-INTERVAL '90d' THEN '90d' ELSE 'older' END AS range, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY MIN(NOW()-created_at)) t),
'updated', (SELECT json_agg(row_to_json(t)) FROM (
SELECT CASE WHEN updated_at>NOW()-INTERVAL '7d' THEN '7d' WHEN updated_at>NOW()-INTERVAL '30d' THEN '30d'
WHEN updated_at>NOW()-INTERVAL '90d' THEN '90d' ELSE 'older' END AS range, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY MIN(NOW()-updated_at)) t)
) AS data
),
top_owners AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT github_owner AS owner, COUNT(*)::int AS skills, COUNT(DISTINCT github_repo)::int AS repos,
MAX(github_stars)::int AS max_stars, COALESCE(SUM(download_count),0)::int AS dl
FROM active GROUP BY github_owner ORDER BY skills DESC LIMIT 30
) t
),
owner_concentration AS (
SELECT json_build_object(
'top1', SUM(cnt) FILTER (WHERE rn<=1)::int, 'top5', SUM(cnt) FILTER (WHERE rn<=5)::int,
'top10', SUM(cnt) FILTER (WHERE rn<=10)::int, 'top20', SUM(cnt) FILTER (WHERE rn<=20)::int,
'total', SUM(cnt)::int, 'owners', COUNT(*)::int
) AS data FROM (
SELECT github_owner, COUNT(*)::int AS cnt, ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn
FROM active GROUP BY github_owner
) x
),
aggregators AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT github_owner||'/'||github_repo AS repo, COUNT(*)::int AS skills, MAX(github_stars)::int AS stars
FROM active GROUP BY github_owner, github_repo HAVING COUNT(*)>=20 ORDER BY skills DESC LIMIT 20
) t
),
collections AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT github_owner||'/'||github_repo AS repo, COUNT(*)::int AS skills, MAX(github_stars)::int AS stars
FROM active GROUP BY github_owner, github_repo HAVING COUNT(*) BETWEEN 3 AND 19 ORDER BY skills DESC LIMIT 20
) t
),
skills_per_repo AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
WITH rc AS (SELECT COUNT(*)::int AS cnt FROM active GROUP BY github_owner, github_repo)
SELECT CASE WHEN cnt=1 THEN '1' WHEN cnt=2 THEN '2' WHEN cnt BETWEEN 3 AND 10 THEN '3-10'
WHEN cnt BETWEEN 11 AND 50 THEN '11-50' ELSE '50+' END AS per_repo,
COUNT(*)::int AS repos, SUM(cnt)::int AS total_skills
FROM rc GROUP BY 1 ORDER BY MIN(cnt)
) t
),
categories_dist AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT c.name AS category, c.skill_count::int AS skills
FROM categories c WHERE c.id NOT LIKE 'parent-%' ORDER BY c.skill_count DESC
) t
),
flags AS (
SELECT json_build_object(
'verified', COUNT(*) FILTER (WHERE is_verified)::int,
'featured', COUNT(*) FILTER (WHERE is_featured)::int,
'user_rated', COUNT(*) FILTER (WHERE rating_count>0)::int
) AS data FROM active
),
cross_stats AS (
SELECT json_build_object(
'users', (SELECT COUNT(*)::int FROM users),
'ratings', (SELECT COUNT(*)::int FROM ratings),
'installs', (SELECT COUNT(*)::int FROM installations),
'favorites', (SELECT COUNT(*)::int FROM favorites),
'disc_repos', (SELECT COUNT(*)::int FROM discovered_repos),
'subs', (SELECT COUNT(*) FILTER (WHERE unsubscribed_at IS NULL)::int FROM email_subscriptions)
) AS data
),
usability AS (
SELECT json_build_object(
'strong', COUNT(*) FILTER (WHERE LENGTH(description)>50 AND raw_content IS NOT NULL AND LENGTH(raw_content)>500 AND security_status='pass')::int,
'downloaded', COUNT(*) FILTER (WHERE COALESCE(download_count,0)>0)::int,
'high_q', COUNT(*) FILTER (WHERE github_stars>=10 AND LENGTH(description)>50 AND raw_content IS NOT NULL AND security_status='pass')::int,
'premium', COUNT(*) FILTER (WHERE github_stars>=100 AND COALESCE(download_count,0)>0 AND security_status='pass')::int,
'skillmd_q', COUNT(*) FILTER (WHERE source_format='skill.md' AND LENGTH(description)>50 AND raw_content IS NOT NULL AND LENGTH(raw_content)>300 AND security_status='pass')::int
) AS data FROM active
),
repo_types AS (
SELECT json_build_object(
'single', COUNT(*) FILTER (WHERE cnt=1)::int, 'two', COUNT(*) FILTER (WHERE cnt=2)::int,
'multi', COUNT(*) FILTER (WHERE cnt>=3)::int, 'total', COUNT(*)::int
) AS data FROM (SELECT COUNT(*)::int AS cnt FROM active GROUP BY github_owner, github_repo) x
),
name_patterns AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT CASE
WHEN name ILIKE '%rule%' OR name ILIKE '%config%' OR name ILIKE '%setup%' THEN 'rules/config'
WHEN name ILIKE '%cursor%' OR name ILIKE '%claude%' OR name ILIKE '%copilot%' THEN 'tool-specific'
WHEN name ILIKE '%project%' OR name ILIKE '%team%' THEN 'project/team'
ELSE 'generic'
END AS pattern, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY skills DESC
) t
),
top_by_stars AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT id, name, github_stars AS stars, COALESCE(download_count,0) AS dl,
security_status AS sec, source_format AS fmt, LEFT(description,80) AS description
FROM active ORDER BY github_stars DESC LIMIT 15
) t
),
discovered AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT discovered_via AS source, COUNT(*)::int AS total,
COUNT(*) FILTER (WHERE has_skill_md)::int AS with_skills
FROM discovered_repos GROUP BY discovered_via ORDER BY total DESC
) t
),
hash_dupes AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT content_hash, COUNT(*)::int AS copies, MIN(name) AS sample
FROM active WHERE content_hash IS NOT NULL
GROUP BY content_hash HAVING COUNT(*)>2 ORDER BY copies DESC LIMIT 15
) t
)
SELECT json_build_object(
'overall', (SELECT data FROM overall),
'sourceFormats', (SELECT data FROM source_formats),
'starsDist', (SELECT data FROM stars_dist),
'dlDist', (SELECT data FROM dl_dist),
'topDL', (SELECT data FROM top_dl),
'topVW', (SELECT data FROM top_vw),
'security', (SELECT data FROM security),
'contentStats', (SELECT data FROM content_stats),
'contentLen', (SELECT data FROM content_len),
'triggerStats', (SELECT data FROM trigger_stats),
'freshness', (SELECT data FROM freshness),
'topOwners', (SELECT data FROM top_owners),
'ownerConcentration', (SELECT data FROM owner_concentration),
'aggregators', (SELECT data FROM aggregators),
'collections', (SELECT data FROM collections),
'skillsPerRepo', (SELECT data FROM skills_per_repo),
'categories', (SELECT data FROM categories_dist),
'flags', (SELECT data FROM flags),
'crossStats', (SELECT data FROM cross_stats),
'usability', (SELECT data FROM usability),
'repoTypes', (SELECT data FROM repo_types),
'namePatterns', (SELECT data FROM name_patterns),
'topByStars', (SELECT data FROM top_by_stars),
'discovered', (SELECT data FROM discovered),
'hashDupes', (SELECT data FROM hash_dupes)
) AS report;

894
scripts/curation/explore.ts
View File

@@ -1,894 +0,0 @@
#!/usr/bin/env tsx
/**
* Phase 1: Database Exploration for Curation
*
* Comprehensive analysis of all indexed skills to understand:
* - Overall statistics and distribution
* - Quality signals and content analysis
* - Owner/repo concentration
* - Freshness and activity
* - Usability signals for curation decisions
*
* Usage:
* cd services/indexer && npx tsx ../../scripts/curation/explore.ts
*
* Output: prints report to console + saves JSON to scripts/curation/explore-report.json
*/
import { createDb, closeDb, sql } from '@skillhub/db';
import { writeFileSync } from 'fs';
import { resolve, dirname } from 'path';
import { fileURLToPath } from 'url';
// ─── Connection ────────────────────────────────────────────────────────────────
const db = createDb();
// ─── Helpers ───────────────────────────────────────────────────────────────────
function header(title: string) {
const line = '='.repeat(70);
console.log(`\n${line}`);
console.log(` ${title}`);
console.log(line);
}
function subHeader(title: string) {
console.log(`\n -- ${title} ${'─'.repeat(Math.max(0, 60 - title.length))}`);
}
function table(rows: Record<string, unknown>[], maxRows = 30) {
if (rows.length === 0) {
console.log(' (no data)');
return;
}
const display = rows.slice(0, maxRows);
const keys = Object.keys(display[0]);
const widths = keys.map(k =>
Math.max(k.length, ...display.map(r => String(r[k] ?? '').length))
);
// Header
console.log(' ' + keys.map((k, i) => k.padEnd(widths[i])).join(' '));
console.log(' ' + widths.map(w => '-'.repeat(w)).join(' '));
// Rows
for (const row of display) {
console.log(' ' + keys.map((k, i) => String(row[k] ?? '').padEnd(widths[i])).join(' '));
}
if (rows.length > maxRows) {
console.log(` ... and ${rows.length - maxRows} more rows`);
}
}
function num(n: unknown): string {
return Number(n ?? 0).toLocaleString('en-US');
}
function pct(part: unknown, total: unknown): string {
const p = Number(part ?? 0);
const t = Number(total ?? 1);
if (t === 0) return '0.0%';
return (p / t * 100).toFixed(1) + '%';
}
// Helper: execute raw SQL and return rows
async function query<T = Record<string, unknown>>(strings: TemplateStringsArray, ...values: unknown[]): Promise<T[]> {
const result = await db.execute(sql.raw(String.raw(strings, ...values)));
return result.rows as T[];
}
// ─── Report accumulator ────────────────────────────────────────────────────────
const report: Record<string, unknown> = {};
// ─── Queries ───────────────────────────────────────────────────────────────────
async function runExploration() {
const startTime = Date.now();
// ======================================================================
// 1. OVERALL STATISTICS
// ======================================================================
header('1. OVERALL STATISTICS');
const overallRows = await db.execute(sql`
SELECT
COUNT(*)::int AS total_skills,
COUNT(*) FILTER (WHERE is_blocked = false)::int AS active_skills,
COUNT(*) FILTER (WHERE is_blocked = true)::int AS blocked_skills,
COUNT(DISTINCT github_owner)::int AS unique_owners,
COUNT(DISTINCT github_owner || '/' || github_repo)::int AS unique_repos,
COUNT(*) FILTER (WHERE source_format = 'skill.md')::int AS skill_md_count,
COUNT(*) FILTER (WHERE source_format != 'skill.md')::int AS other_format_count,
COALESCE(SUM(download_count), 0)::bigint AS total_downloads,
COALESCE(SUM(view_count), 0)::bigint AS total_views,
COUNT(*) FILTER (WHERE rating_count > 0)::int AS rated_skills,
COALESCE(SUM(rating_count), 0)::int AS total_ratings,
MIN(created_at)::text AS earliest_skill,
MAX(created_at)::text AS latest_skill
FROM skills
`);
const overall = overallRows.rows[0] as Record<string, unknown>;
report.overall = overall;
console.log(` Total skills: ${num(overall.total_skills)}`);
console.log(` Active (not blocked): ${num(overall.active_skills)}`);
console.log(` Blocked: ${num(overall.blocked_skills)}`);
console.log(` Unique owners: ${num(overall.unique_owners)}`);
console.log(` Unique repos: ${num(overall.unique_repos)}`);
console.log(` SKILL.md format: ${num(overall.skill_md_count)} (${pct(overall.skill_md_count, overall.total_skills)})`);
console.log(` Other formats: ${num(overall.other_format_count)} (${pct(overall.other_format_count, overall.total_skills)})`);
console.log(` Total downloads: ${num(overall.total_downloads)}`);
console.log(` Total views: ${num(overall.total_views)}`);
console.log(` Rated skills: ${num(overall.rated_skills)}`);
console.log(` Total ratings: ${num(overall.total_ratings)}`);
console.log(` Date range: ${overall.earliest_skill} -> ${overall.latest_skill}`);
const total = Number(overall.active_skills);
// Source format distribution
subHeader('Source Format Distribution');
const sfRows = await db.execute(sql`
SELECT
COALESCE(source_format, 'null') AS format,
COUNT(*)::int AS count,
COUNT(*) FILTER (WHERE is_blocked = false)::int AS active
FROM skills
GROUP BY source_format
ORDER BY count DESC
`);
report.sourceFormats = sfRows.rows;
table(sfRows.rows as Record<string, unknown>[]);
// ======================================================================
// 2. STARS DISTRIBUTION
// ======================================================================
header('2. GITHUB STARS DISTRIBUTION');
const starsRows = await db.execute(sql`
SELECT
CASE
WHEN github_stars IS NULL OR github_stars = 0 THEN '0 stars'
WHEN github_stars BETWEEN 1 AND 5 THEN '1-5'
WHEN github_stars BETWEEN 6 AND 10 THEN '6-10'
WHEN github_stars BETWEEN 11 AND 50 THEN '11-50'
WHEN github_stars BETWEEN 51 AND 100 THEN '51-100'
WHEN github_stars BETWEEN 101 AND 500 THEN '101-500'
WHEN github_stars BETWEEN 501 AND 1000 THEN '501-1K'
WHEN github_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
WHEN github_stars BETWEEN 5001 AND 10000 THEN '5K-10K'
WHEN github_stars BETWEEN 10001 AND 50000 THEN '10K-50K'
ELSE '50K+'
END AS star_range,
COUNT(*)::int AS skills,
COUNT(DISTINCT github_owner)::int AS owners,
COUNT(DISTINCT github_owner || '/' || github_repo)::int AS repos
FROM skills
WHERE is_blocked = false
GROUP BY
CASE
WHEN github_stars IS NULL OR github_stars = 0 THEN 0
WHEN github_stars BETWEEN 1 AND 5 THEN 1
WHEN github_stars BETWEEN 6 AND 10 THEN 2
WHEN github_stars BETWEEN 11 AND 50 THEN 3
WHEN github_stars BETWEEN 51 AND 100 THEN 4
WHEN github_stars BETWEEN 101 AND 500 THEN 5
WHEN github_stars BETWEEN 501 AND 1000 THEN 6
WHEN github_stars BETWEEN 1001 AND 5000 THEN 7
WHEN github_stars BETWEEN 5001 AND 10000 THEN 8
WHEN github_stars BETWEEN 10001 AND 50000 THEN 9
ELSE 10
END,
CASE
WHEN github_stars IS NULL OR github_stars = 0 THEN '0 stars'
WHEN github_stars BETWEEN 1 AND 5 THEN '1-5'
WHEN github_stars BETWEEN 6 AND 10 THEN '6-10'
WHEN github_stars BETWEEN 11 AND 50 THEN '11-50'
WHEN github_stars BETWEEN 51 AND 100 THEN '51-100'
WHEN github_stars BETWEEN 101 AND 500 THEN '101-500'
WHEN github_stars BETWEEN 501 AND 1000 THEN '501-1K'
WHEN github_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
WHEN github_stars BETWEEN 5001 AND 10000 THEN '5K-10K'
WHEN github_stars BETWEEN 10001 AND 50000 THEN '10K-50K'
ELSE '50K+'
END
ORDER BY MIN(COALESCE(github_stars, 0))
`);
report.starsDist = starsRows.rows;
table(starsRows.rows as Record<string, unknown>[]);
// ======================================================================
// 3. ENGAGEMENT DISTRIBUTION
// ======================================================================
header('3. ENGAGEMENT (Downloads & Views)');
subHeader('Download Distribution');
const dlRows = await db.execute(sql`
SELECT
CASE
WHEN COALESCE(download_count, 0) = 0 THEN '0'
WHEN download_count BETWEEN 1 AND 5 THEN '1-5'
WHEN download_count BETWEEN 6 AND 10 THEN '6-10'
WHEN download_count BETWEEN 11 AND 50 THEN '11-50'
WHEN download_count BETWEEN 51 AND 100 THEN '51-100'
WHEN download_count BETWEEN 101 AND 500 THEN '101-500'
ELSE '500+'
END AS range,
COUNT(*)::int AS skills
FROM skills WHERE is_blocked = false
GROUP BY 1 ORDER BY MIN(COALESCE(download_count, 0))
`);
report.downloadDist = dlRows.rows;
table(dlRows.rows as Record<string, unknown>[]);
subHeader('View Distribution');
const vwRows = await db.execute(sql`
SELECT
CASE
WHEN COALESCE(view_count, 0) = 0 THEN '0'
WHEN view_count BETWEEN 1 AND 10 THEN '1-10'
WHEN view_count BETWEEN 11 AND 50 THEN '11-50'
WHEN view_count BETWEEN 51 AND 100 THEN '51-100'
WHEN view_count BETWEEN 101 AND 500 THEN '101-500'
WHEN view_count BETWEEN 501 AND 1000 THEN '501-1K'
ELSE '1K+'
END AS range,
COUNT(*)::int AS skills
FROM skills WHERE is_blocked = false
GROUP BY 1 ORDER BY MIN(COALESCE(view_count, 0))
`);
report.viewDist = vwRows.rows;
table(vwRows.rows as Record<string, unknown>[]);
subHeader('Top 20 Downloaded Skills');
const topDL = await db.execute(sql`
SELECT id, name, download_count AS downloads, view_count AS views,
github_stars AS stars, github_owner AS owner
FROM skills
WHERE is_blocked = false AND COALESCE(download_count, 0) > 0
ORDER BY download_count DESC LIMIT 20
`);
report.topDownloads = topDL.rows;
table(topDL.rows as Record<string, unknown>[]);
subHeader('Top 20 Viewed Skills');
const topVW = await db.execute(sql`
SELECT id, name, view_count AS views, download_count AS downloads,
github_stars AS stars, github_owner AS owner
FROM skills
WHERE is_blocked = false AND COALESCE(view_count, 0) > 0
ORDER BY view_count DESC LIMIT 20
`);
report.topViews = topVW.rows;
table(topVW.rows as Record<string, unknown>[]);
// ======================================================================
// 4. SECURITY STATUS
// ======================================================================
header('4. SECURITY STATUS');
const secRows = await db.execute(sql`
SELECT
COALESCE(security_status, 'null') AS status,
COUNT(*)::int AS count,
ROUND(COUNT(*)::numeric / NULLIF(${total}, 0) * 100, 1) AS pct
FROM skills WHERE is_blocked = false
GROUP BY security_status ORDER BY count DESC
`);
report.securityDist = secRows.rows;
table(secRows.rows as Record<string, unknown>[]);
// ======================================================================
// 5. CONTENT ANALYSIS
// ======================================================================
header('5. CONTENT ANALYSIS');
const csRows = await db.execute(sql`
SELECT
COUNT(*) FILTER (WHERE raw_content IS NOT NULL)::int AS has_content,
COUNT(*) FILTER (WHERE raw_content IS NULL)::int AS no_content,
ROUND(AVG(LENGTH(raw_content)) FILTER (WHERE raw_content IS NOT NULL))::int AS avg_content_len,
MAX(LENGTH(raw_content))::int AS max_content_len,
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY LENGTH(raw_content))
FILTER (WHERE raw_content IS NOT NULL)::int AS median_content_len,
ROUND(AVG(LENGTH(description)))::int AS avg_desc_len,
COUNT(*) FILTER (WHERE LENGTH(description) > 100)::int AS good_desc,
COUNT(*) FILTER (WHERE LENGTH(description) <= 20)::int AS short_desc,
COUNT(*) FILTER (WHERE version IS NOT NULL)::int AS has_version,
COUNT(*) FILTER (WHERE license IS NOT NULL)::int AS has_license,
COUNT(*) FILTER (WHERE author IS NOT NULL)::int AS has_author,
COUNT(*) FILTER (WHERE homepage IS NOT NULL)::int AS has_homepage
FROM skills WHERE is_blocked = false
`);
const cs = csRows.rows[0] as Record<string, unknown>;
report.contentStats = cs;
console.log(` Has raw_content: ${num(cs.has_content)} (${pct(cs.has_content, total)})`);
console.log(` No raw_content: ${num(cs.no_content)} (${pct(cs.no_content, total)})`);
console.log(` Avg content length: ${num(cs.avg_content_len)} chars`);
console.log(` Median content len: ${num(cs.median_content_len)} chars`);
console.log(` Max content length: ${num(cs.max_content_len)} chars`);
console.log(` Avg desc length: ${num(cs.avg_desc_len)} chars`);
console.log(` Good desc (>100ch): ${num(cs.good_desc)} (${pct(cs.good_desc, total)})`);
console.log(` Short desc (<=20): ${num(cs.short_desc)} (${pct(cs.short_desc, total)})`);
console.log(` Has version: ${num(cs.has_version)} (${pct(cs.has_version, total)})`);
console.log(` Has license: ${num(cs.has_license)} (${pct(cs.has_license, total)})`);
console.log(` Has author: ${num(cs.has_author)} (${pct(cs.has_author, total)})`);
console.log(` Has homepage: ${num(cs.has_homepage)} (${pct(cs.has_homepage, total)})`);
subHeader('Content Length Distribution');
const clRows = await db.execute(sql`
SELECT
CASE
WHEN raw_content IS NULL THEN 'null'
WHEN LENGTH(raw_content) = 0 THEN 'empty'
WHEN LENGTH(raw_content) < 200 THEN '<200'
WHEN LENGTH(raw_content) < 500 THEN '200-500'
WHEN LENGTH(raw_content) < 1000 THEN '500-1K'
WHEN LENGTH(raw_content) < 3000 THEN '1K-3K'
WHEN LENGTH(raw_content) < 5000 THEN '3K-5K'
WHEN LENGTH(raw_content) < 10000 THEN '5K-10K'
WHEN LENGTH(raw_content) < 50000 THEN '10K-50K'
ELSE '50K+'
END AS range,
COUNT(*)::int AS skills
FROM skills WHERE is_blocked = false
GROUP BY 1 ORDER BY MIN(COALESCE(LENGTH(raw_content), -1))
`);
report.contentLenDist = clRows.rows;
table(clRows.rows as Record<string, unknown>[]);
// ======================================================================
// 6. TRIGGERS & COMPATIBILITY
// ======================================================================
header('6. TRIGGERS & COMPATIBILITY');
const trRows = await db.execute(sql`
SELECT
COUNT(*) FILTER (WHERE triggers IS NOT NULL)::int AS has_triggers,
COUNT(*) FILTER (WHERE triggers IS NULL)::int AS no_triggers,
COUNT(*) FILTER (WHERE triggers->>'filePatterns' IS NOT NULL
AND triggers->>'filePatterns' != '[]'
AND triggers->>'filePatterns' != 'null')::int AS has_file_patterns,
COUNT(*) FILTER (WHERE triggers->>'keywords' IS NOT NULL
AND triggers->>'keywords' != '[]'
AND triggers->>'keywords' != 'null')::int AS has_keywords,
COUNT(*) FILTER (WHERE triggers->>'languages' IS NOT NULL
AND triggers->>'languages' != '[]'
AND triggers->>'languages' != 'null')::int AS has_languages,
COUNT(*) FILTER (WHERE compatibility IS NOT NULL)::int AS has_compat,
COUNT(*) FILTER (WHERE compatibility->>'platforms' IS NOT NULL
AND compatibility->>'platforms' != '[]'
AND compatibility->>'platforms' != 'null')::int AS has_platforms,
COUNT(*) FILTER (WHERE compatibility->>'requires' IS NOT NULL
AND compatibility->>'requires' != '[]'
AND compatibility->>'requires' != 'null')::int AS has_requires
FROM skills WHERE is_blocked = false
`);
const tr = trRows.rows[0] as Record<string, unknown>;
report.triggerStats = tr;
console.log(` Has triggers: ${num(tr.has_triggers)} (${pct(tr.has_triggers, total)})`);
console.log(` No triggers: ${num(tr.no_triggers)} (${pct(tr.no_triggers, total)})`);
console.log(` - filePatterns: ${num(tr.has_file_patterns)}`);
console.log(` - keywords: ${num(tr.has_keywords)}`);
console.log(` - languages: ${num(tr.has_languages)}`);
console.log(` Has compatibility: ${num(tr.has_compat)}`);
console.log(` - platforms: ${num(tr.has_platforms)}`);
console.log(` - requires: ${num(tr.has_requires)}`);
// ======================================================================
// 7. FRESHNESS
// ======================================================================
header('7. FRESHNESS & ACTIVITY');
subHeader('Created At Distribution');
const crRows = await db.execute(sql`
SELECT
CASE
WHEN created_at > NOW() - INTERVAL '7 days' THEN 'Last 7d'
WHEN created_at > NOW() - INTERVAL '30 days' THEN 'Last 30d'
WHEN created_at > NOW() - INTERVAL '90 days' THEN 'Last 90d'
WHEN created_at > NOW() - INTERVAL '180 days' THEN 'Last 180d'
WHEN created_at > NOW() - INTERVAL '365 days' THEN 'Last 365d'
ELSE 'Older'
END AS range,
COUNT(*)::int AS skills
FROM skills WHERE is_blocked = false
GROUP BY 1 ORDER BY MIN(NOW() - created_at)
`);
report.createdDist = crRows.rows;
table(crRows.rows as Record<string, unknown>[]);
subHeader('Updated At Distribution');
const upRows = await db.execute(sql`
SELECT
CASE
WHEN updated_at > NOW() - INTERVAL '7 days' THEN 'Last 7d'
WHEN updated_at > NOW() - INTERVAL '30 days' THEN 'Last 30d'
WHEN updated_at > NOW() - INTERVAL '90 days' THEN 'Last 90d'
WHEN updated_at > NOW() - INTERVAL '180 days' THEN 'Last 180d'
WHEN updated_at > NOW() - INTERVAL '365 days' THEN 'Last 365d'
ELSE 'Older'
END AS range,
COUNT(*)::int AS skills
FROM skills WHERE is_blocked = false
GROUP BY 1 ORDER BY MIN(NOW() - updated_at)
`);
report.updatedDist = upRows.rows;
table(upRows.rows as Record<string, unknown>[]);
subHeader('Last Downloaded Distribution');
const ldRows = await db.execute(sql`
SELECT
CASE
WHEN last_downloaded_at IS NULL THEN 'Never'
WHEN last_downloaded_at > NOW() - INTERVAL '7 days' THEN 'Last 7d'
WHEN last_downloaded_at > NOW() - INTERVAL '30 days' THEN 'Last 30d'
WHEN last_downloaded_at > NOW() - INTERVAL '90 days' THEN 'Last 90d'
ELSE 'Older'
END AS range,
COUNT(*)::int AS skills
FROM skills WHERE is_blocked = false
GROUP BY 1 ORDER BY MIN(COALESCE(last_downloaded_at, '1970-01-01'::timestamp))
`);
report.lastDownDist = ldRows.rows;
table(ldRows.rows as Record<string, unknown>[]);
// ======================================================================
// 8. TOP OWNERS
// ======================================================================
header('8. TOP OWNERS');
subHeader('Top 30 Owners by Skill Count');
const owRows = await db.execute(sql`
SELECT
github_owner AS owner,
COUNT(*)::int AS skills,
COUNT(DISTINCT github_repo)::int AS repos,
MAX(github_stars)::int AS max_stars,
COALESCE(SUM(download_count), 0)::int AS downloads,
COALESCE(SUM(view_count), 0)::int AS views
FROM skills WHERE is_blocked = false
GROUP BY github_owner ORDER BY skills DESC LIMIT 30
`);
report.topOwnersByCount = owRows.rows;
table(owRows.rows as Record<string, unknown>[]);
subHeader('Top 20 Owners by Stars');
const osRows = await db.execute(sql`
SELECT
github_owner AS owner,
MAX(github_stars)::int AS max_stars,
COUNT(*)::int AS skills,
COUNT(DISTINCT github_repo)::int AS repos,
COALESCE(SUM(download_count), 0)::int AS downloads
FROM skills WHERE is_blocked = false
GROUP BY github_owner ORDER BY max_stars DESC LIMIT 20
`);
report.topOwnersByStars = osRows.rows;
table(osRows.rows as Record<string, unknown>[]);
subHeader('Owner Concentration');
const ocRows = await db.execute(sql`
WITH ranked AS (
SELECT github_owner, COUNT(*)::int AS cnt,
ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn
FROM skills WHERE is_blocked = false
GROUP BY github_owner
)
SELECT
SUM(cnt) FILTER (WHERE rn <= 1)::int AS top_1,
SUM(cnt) FILTER (WHERE rn <= 5)::int AS top_5,
SUM(cnt) FILTER (WHERE rn <= 10)::int AS top_10,
SUM(cnt) FILTER (WHERE rn <= 20)::int AS top_20,
SUM(cnt) FILTER (WHERE rn <= 50)::int AS top_50,
SUM(cnt)::int AS total,
COUNT(*)::int AS owners
FROM ranked
`);
const oc = ocRows.rows[0] as Record<string, unknown>;
report.ownerConcentration = oc;
console.log(` Top 1 owner: ${num(oc.top_1)} skills (${pct(oc.top_1, oc.total)})`);
console.log(` Top 5 owners: ${num(oc.top_5)} skills (${pct(oc.top_5, oc.total)})`);
console.log(` Top 10 owners: ${num(oc.top_10)} skills (${pct(oc.top_10, oc.total)})`);
console.log(` Top 20 owners: ${num(oc.top_20)} skills (${pct(oc.top_20, oc.total)})`);
console.log(` Top 50 owners: ${num(oc.top_50)} skills (${pct(oc.top_50, oc.total)})`);
console.log(` Total owners: ${num(oc.owners)}`);
// ======================================================================
// 9. MULTI-SKILL REPOS
// ======================================================================
header('9. MULTI-SKILL REPOS');
subHeader('Repos with 20+ Skills (Aggregator Candidates)');
const aggRows = await db.execute(sql`
SELECT
github_owner || '/' || github_repo AS repo,
COUNT(*)::int AS skills,
MAX(github_stars)::int AS stars,
COALESCE(SUM(download_count), 0)::int AS downloads
FROM skills WHERE is_blocked = false
GROUP BY github_owner, github_repo
HAVING COUNT(*) >= 20
ORDER BY skills DESC LIMIT 30
`);
report.aggregatorCandidates = aggRows.rows;
table(aggRows.rows as Record<string, unknown>[]);
subHeader('Repos with 3-19 Skills (Collection Candidates)');
const colRows = await db.execute(sql`
SELECT
github_owner || '/' || github_repo AS repo,
COUNT(*)::int AS skills,
MAX(github_stars)::int AS stars,
COALESCE(SUM(download_count), 0)::int AS downloads
FROM skills WHERE is_blocked = false
GROUP BY github_owner, github_repo
HAVING COUNT(*) BETWEEN 3 AND 19
ORDER BY skills DESC LIMIT 30
`);
report.collectionCandidates = colRows.rows;
table(colRows.rows as Record<string, unknown>[]);
subHeader('Skills-per-Repo Distribution');
const sprRows = await db.execute(sql`
WITH rc AS (
SELECT COUNT(*)::int AS cnt
FROM skills WHERE is_blocked = false
GROUP BY github_owner, github_repo
)
SELECT
CASE
WHEN cnt = 1 THEN '1 skill'
WHEN cnt = 2 THEN '2 skills'
WHEN cnt BETWEEN 3 AND 5 THEN '3-5'
WHEN cnt BETWEEN 6 AND 10 THEN '6-10'
WHEN cnt BETWEEN 11 AND 20 THEN '11-20'
WHEN cnt BETWEEN 21 AND 50 THEN '21-50'
WHEN cnt BETWEEN 51 AND 100 THEN '51-100'
ELSE '100+'
END AS per_repo,
COUNT(*)::int AS repos,
SUM(cnt)::int AS total_skills
FROM rc
GROUP BY 1 ORDER BY MIN(cnt)
`);
report.skillsPerRepo = sprRows.rows;
table(sprRows.rows as Record<string, unknown>[]);
// ======================================================================
// 10. CATEGORY DISTRIBUTION
// ======================================================================
header('10. CATEGORY DISTRIBUTION');
const catRows = await db.execute(sql`
SELECT c.name AS category, c.skill_count::int AS skills, c.id
FROM categories c
WHERE c.id NOT LIKE 'parent-%'
ORDER BY c.skill_count DESC
`);
report.categoryDist = catRows.rows;
table(catRows.rows as Record<string, unknown>[]);
const ncRows = await db.execute(sql`
SELECT COUNT(*)::int AS count
FROM skills s
WHERE s.is_blocked = false
AND NOT EXISTS (SELECT 1 FROM skill_categories sc WHERE sc.skill_id = s.id)
`);
report.uncategorizedCount = (ncRows.rows[0] as Record<string, unknown>)?.count;
console.log(`\n Skills with no category: ${num(report.uncategorizedCount)}`);
// ======================================================================
// 11. FLAGS & RATINGS
// ======================================================================
header('11. FLAGS & RATINGS');
const flRows = await db.execute(sql`
SELECT
COUNT(*) FILTER (WHERE is_verified = true)::int AS verified,
COUNT(*) FILTER (WHERE is_featured = true)::int AS featured,
COUNT(*) FILTER (WHERE rating IS NOT NULL)::int AS has_rating,
COUNT(*) FILTER (WHERE rating_count > 0)::int AS has_user_ratings,
ROUND(AVG(rating) FILTER (WHERE rating IS NOT NULL), 2)::numeric AS avg_rating,
MAX(rating_count)::int AS max_rating_count
FROM skills WHERE is_blocked = false
`);
const fl = flRows.rows[0] as Record<string, unknown>;
report.flags = fl;
console.log(` Verified: ${num(fl.verified)}`);
console.log(` Featured: ${num(fl.featured)}`);
console.log(` Has rating: ${num(fl.has_rating)}`);
console.log(` Has user ratings: ${num(fl.has_user_ratings)}`);
console.log(` Average rating: ${fl.avg_rating ?? 'N/A'}`);
console.log(` Max rating count: ${num(fl.max_rating_count)}`);
// ======================================================================
// 12. CROSS-TABLE STATS
// ======================================================================
header('12. CROSS-TABLE STATS');
const xtRows = await db.execute(sql`
SELECT
(SELECT COUNT(*)::int FROM users) AS users,
(SELECT COUNT(*)::int FROM ratings) AS ratings,
(SELECT COUNT(*)::int FROM installations) AS installations,
(SELECT COUNT(*)::int FROM favorites) AS favorites,
(SELECT COUNT(*)::int FROM discovered_repos) AS discovered_repos,
(SELECT COUNT(*) FILTER (WHERE has_skill_md = true)::int FROM discovered_repos) AS repos_with_skills,
(SELECT COUNT(*)::int FROM removal_requests) AS removal_requests,
(SELECT COUNT(*)::int FROM add_requests) AS add_requests,
(SELECT COUNT(*) FILTER (WHERE unsubscribed_at IS NULL)::int FROM email_subscriptions) AS subscribers
`);
const xt = xtRows.rows[0] as Record<string, unknown>;
report.crossStats = xt;
console.log(` Users: ${num(xt.users)}`);
console.log(` Ratings: ${num(xt.ratings)}`);
console.log(` Installations: ${num(xt.installations)}`);
console.log(` Favorites: ${num(xt.favorites)}`);
console.log(` Discovered repos: ${num(xt.discovered_repos)}`);
console.log(` - with skills: ${num(xt.repos_with_skills)}`);
console.log(` Removal requests: ${num(xt.removal_requests)}`);
console.log(` Add requests: ${num(xt.add_requests)}`);
console.log(` Subscribers: ${num(xt.subscribers)}`);
subHeader('Installations by Platform');
const ipRows = await db.execute(sql`
SELECT platform, COUNT(*)::int AS count
FROM installations GROUP BY platform ORDER BY count DESC
`);
report.installByPlatform = ipRows.rows;
table(ipRows.rows as Record<string, unknown>[]);
// ======================================================================
// 13. USABILITY SIGNALS
// ======================================================================
header('13. USABILITY SIGNALS FOR CURATION');
const usRows = await db.execute(sql`
SELECT
COUNT(*) FILTER (
WHERE LENGTH(description) > 50
AND raw_content IS NOT NULL AND LENGTH(raw_content) > 500
AND security_status = 'pass'
)::int AS strong_standalone,
COUNT(*) FILTER (
WHERE triggers IS NOT NULL AND triggers != '{}'::jsonb
AND (triggers->>'filePatterns' IS NOT NULL AND triggers->>'filePatterns' != '[]')
)::int AS has_file_triggers,
COUNT(*) FILTER (WHERE COALESCE(download_count, 0) > 0)::int AS ever_downloaded,
COUNT(*) FILTER (
WHERE github_stars >= 10
AND LENGTH(description) > 50
AND raw_content IS NOT NULL
AND security_status = 'pass'
)::int AS high_quality,
COUNT(*) FILTER (
WHERE github_stars >= 100
AND COALESCE(download_count, 0) > 0
AND security_status = 'pass'
)::int AS premium,
COUNT(*) FILTER (
WHERE source_format = 'skill.md'
AND LENGTH(description) > 50
AND raw_content IS NOT NULL AND LENGTH(raw_content) > 300
AND security_status = 'pass'
)::int AS skillmd_quality
FROM skills WHERE is_blocked = false
`);
const us = usRows.rows[0] as Record<string, unknown>;
report.usability = us;
console.log(` Strong standalone candidates: ${num(us.strong_standalone)}`);
console.log(` (desc>50 + content>500 + security=pass)`);
console.log(` With file triggers: ${num(us.has_file_triggers)}`);
console.log(` Ever downloaded: ${num(us.ever_downloaded)}`);
console.log(` High quality (stars>10+desc+sec): ${num(us.high_quality)}`);
console.log(` Premium (stars>100+dl>0+sec): ${num(us.premium)}`);
console.log(` SKILL.md with quality: ${num(us.skillmd_quality)}`);
// ======================================================================
// 14. STANDALONE vs PROJECT-BOUND
// ======================================================================
header('14. STANDALONE vs PROJECT-BOUND HEURISTIC');
subHeader('Single vs Multi-Skill Repos');
const smRows = await db.execute(sql`
WITH rc AS (
SELECT github_owner, github_repo, COUNT(*)::int AS cnt
FROM skills WHERE is_blocked = false
GROUP BY github_owner, github_repo
)
SELECT
COUNT(*) FILTER (WHERE cnt = 1)::int AS single,
COUNT(*) FILTER (WHERE cnt = 2)::int AS two,
COUNT(*) FILTER (WHERE cnt >= 3)::int AS multi,
COUNT(*)::int AS total
FROM rc
`);
const sm = smRows.rows[0] as Record<string, unknown>;
report.repoTypes = sm;
console.log(` Single-skill repos: ${num(sm.single)} (${pct(sm.single, sm.total)})`);
console.log(` Two-skill repos: ${num(sm.two)} (${pct(sm.two, sm.total)})`);
console.log(` Multi-skill repos: ${num(sm.multi)} (${pct(sm.multi, sm.total)})`);
subHeader('Name Pattern Signals');
const npRows = await db.execute(sql`
SELECT
CASE
WHEN name ILIKE '%rule%' OR name ILIKE '%config%' OR name ILIKE '%setup%' THEN 'rules/config/setup'
WHEN name ILIKE '%my-%' OR name ILIKE '%my_%' THEN 'starts with my-'
WHEN name ILIKE '%cursor%' OR name ILIKE '%claude%' OR name ILIKE '%copilot%' THEN 'tool-specific'
WHEN name ILIKE '%project%' OR name ILIKE '%team%' OR name ILIKE '%internal%' THEN 'project/team'
WHEN name ILIKE '%.mdc' OR name ILIKE '%cursorrule%' THEN 'cursor rules file'
ELSE 'generic/other'
END AS pattern,
COUNT(*)::int AS skills
FROM skills WHERE is_blocked = false
GROUP BY 1 ORDER BY skills DESC
`);
report.namePatterns = npRows.rows;
table(npRows.rows as Record<string, unknown>[]);
// ======================================================================
// 15. SAMPLE SKILLS
// ======================================================================
header('15. SAMPLE SKILLS');
subHeader('Top 15 by Stars');
const tsRows = await db.execute(sql`
SELECT id, name, github_stars AS stars, COALESCE(download_count,0) AS dl,
security_status AS sec, source_format AS fmt,
LEFT(description, 80) AS description
FROM skills WHERE is_blocked = false
ORDER BY github_stars DESC LIMIT 15
`);
report.topByStars = tsRows.rows;
table(tsRows.rows as Record<string, unknown>[]);
subHeader('Top 15 by Downloads (actually used)');
const tdRows = await db.execute(sql`
SELECT id, name, download_count AS dl, view_count AS views,
github_stars AS stars, security_status AS sec,
LEFT(description, 80) AS description
FROM skills WHERE is_blocked = false AND COALESCE(download_count,0) > 0
ORDER BY download_count DESC LIMIT 15
`);
report.topByDL = tdRows.rows;
table(tdRows.rows as Record<string, unknown>[]);
subHeader('SKILL.md + Stars>50 + Downloads>0');
const qsRows = await db.execute(sql`
SELECT id, name, github_stars AS stars, download_count AS dl,
security_status AS sec, LEFT(description, 80) AS description
FROM skills
WHERE is_blocked = false AND source_format = 'skill.md'
AND github_stars >= 50 AND COALESCE(download_count, 0) > 0
ORDER BY github_stars DESC LIMIT 20
`);
report.qualitySkillMd = qsRows.rows;
table(qsRows.rows as Record<string, unknown>[]);
// ======================================================================
// 16. DISCOVERED REPOS
// ======================================================================
header('16. DISCOVERED REPOS');
const drRows = await db.execute(sql`
SELECT
discovered_via AS source,
COUNT(*)::int AS total,
COUNT(*) FILTER (WHERE has_skill_md = true)::int AS with_skills,
COUNT(*) FILTER (WHERE last_scanned IS NOT NULL)::int AS scanned,
COUNT(*) FILTER (WHERE is_archived = true)::int AS archived
FROM discovered_repos
GROUP BY discovered_via ORDER BY total DESC
`);
report.discoveredStats = drRows.rows;
table(drRows.rows as Record<string, unknown>[]);
// ======================================================================
// 17. CACHED FILES
// ======================================================================
header('17. CACHED FILES');
const cfRows = await db.execute(sql`
SELECT
COUNT(*) FILTER (WHERE cached_files IS NOT NULL)::int AS has_cache,
COUNT(*) FILTER (WHERE cached_files IS NULL)::int AS no_cache
FROM skills WHERE is_blocked = false
`);
const cf = cfRows.rows[0] as Record<string, unknown>;
report.cacheStats = cf;
console.log(` Has cached files: ${num(cf.has_cache)} (${pct(cf.has_cache, total)})`);
console.log(` No cached files: ${num(cf.no_cache)} (${pct(cf.no_cache, total)})`);
// ======================================================================
// 19. BRANCH DISTRIBUTION
// ======================================================================
header('19. BRANCH DISTRIBUTION');
const brRows = await db.execute(sql`
SELECT COALESCE(branch, 'null') AS branch, COUNT(*)::int AS count
FROM skills WHERE is_blocked = false
GROUP BY branch ORDER BY count DESC LIMIT 10
`);
report.branchDist = brRows.rows;
table(brRows.rows as Record<string, unknown>[]);
// ======================================================================
// 20. DUPLICATES
// ======================================================================
header('20. POTENTIAL DUPLICATES');
subHeader('Same Name + Description');
const ddRows = await db.execute(sql`
SELECT name, LEFT(description, 60) AS desc, COUNT(*)::int AS occurrences,
STRING_AGG(DISTINCT github_owner, ', ') AS owners
FROM skills
WHERE is_blocked = false AND description IS NOT NULL AND LENGTH(description) > 10
GROUP BY name, description HAVING COUNT(*) > 1
ORDER BY occurrences DESC LIMIT 15
`);
report.exactDupes = ddRows.rows;
table(ddRows.rows as Record<string, unknown>[]);
subHeader('Same Content Hash (identical content, 3+ copies)');
const chRows = await db.execute(sql`
SELECT content_hash, COUNT(*)::int AS copies,
MIN(name) AS sample_name,
STRING_AGG(DISTINCT github_owner, ', ') AS owners
FROM skills
WHERE is_blocked = false AND content_hash IS NOT NULL
GROUP BY content_hash HAVING COUNT(*) > 2
ORDER BY copies DESC LIMIT 15
`);
report.hashDupes = chRows.rows;
table(chRows.rows as Record<string, unknown>[]);
// ======================================================================
// DONE
// ======================================================================
const duration = ((Date.now() - startTime) / 1000).toFixed(1);
header(`EXPLORATION COMPLETE (${duration}s)`);
console.log(`
KEY QUESTIONS TO ANSWER:
1. What % of skills are SKILL.md vs other formats?
2. What % have been downloaded at least once? (real usage)
3. How concentrated is ownership? (top 10 owners = ?%)
4. How many multi-skill repos? (collection/aggregator candidates)
5. How many pass security + have good content? (curation-ready)
6. How many duplicates exist?
`);
// Save JSON report
report.generatedAt = new Date().toISOString();
report.durationSeconds = parseFloat(duration);
const scriptDir = dirname(fileURLToPath(import.meta.url));
const reportPath = resolve(scriptDir, 'explore-report.json');
writeFileSync(reportPath, JSON.stringify(report, null, 2), 'utf-8');
console.log(` Full report saved to: ${reportPath}`);
await closeDb();
}
// ─── Run ───────────────────────────────────────────────────────────────────────
runExploration().catch(async (err) => {
console.error('Exploration failed:', err);
await closeDb();
process.exit(1);
});

22
scripts/init-db.sql
View File

@@ -493,15 +493,23 @@ ALTER TABLE skills ADD COLUMN IF NOT EXISTS is_deprecated BOOLEAN DEFAULT FALSE;
-- Safe pre-filter function for raw_content (handles invalid UTF-8 gracefully)
-- Used by review pipeline queries (getPending, countPending, countReReviews)
-- NOTE: EXCEPTION handler returns TRUE (benefit of the doubt) — CJK skills with
-- encoding edge cases should not be silently excluded from the review pipeline.
-- Genuine content issues will be caught during the actual review.
CREATE OR REPLACE FUNCTION raw_content_passes_prefilter(content TEXT) RETURNS BOOLEAN AS $$
BEGIN
-- Length check (octet_length works on bytes, safe for any encoding)
IF octet_length(content) < 200 THEN RETURN FALSE; END IF;
IF position('<!-- generated' in content) > 0 THEN RETURN FALSE; END IF;
IF position('/Users/' in LEFT(content, 1000)) > 0 THEN RETURN FALSE; END IF;
IF position('C:\Users\' in LEFT(content, 1000)) > 0 THEN RETURN FALSE; END IF;
-- Content checks (may fail on encoding edge cases)
BEGIN
IF position('<!-- generated' in content) > 0 THEN RETURN FALSE; END IF;
IF position('/Users/' in LEFT(content, 1000)) > 0 THEN RETURN FALSE; END IF;
IF position('C:\Users\' in LEFT(content, 1000)) > 0 THEN RETURN FALSE; END IF;
EXCEPTION WHEN OTHERS THEN
-- Encoding error in string checks — content is long enough, let it through
RETURN TRUE;
END;
RETURN TRUE;
EXCEPTION WHEN OTHERS THEN
RETURN FALSE;
END;
$$ LANGUAGE plpgsql IMMUTABLE;
@@ -573,6 +581,10 @@ ALTER TABLE skills ADD COLUMN IF NOT EXISTS latest_ai_score INTEGER;
ALTER TABLE skills ADD COLUMN IF NOT EXISTS latest_review_date TIMESTAMPTZ;
CREATE INDEX IF NOT EXISTS idx_skills_ai_score ON skills(latest_ai_score) WHERE latest_ai_score IS NOT NULL;
-- Malicious skill detection (March 2026)
ALTER TABLE skills ADD COLUMN IF NOT EXISTS is_malicious BOOLEAN DEFAULT FALSE;
ALTER TABLE skill_reviews ADD COLUMN IF NOT EXISTS recommendation TEXT;
-- Grant permissions
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO postgres;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO postgres;