atlatl/skillhub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(curation): data quality pipeline — Phases 1-3

Browse Source 
Add comprehensive data curation system to clean up the 197K skill
dataset and show only quality browse-ready skills to users.

Phase 1 — Database exploration:
- Explore scripts (explore.ts, explore.mjs, explore.sql) for analysis
- Discovered: 69% duplicates, 77% aggregator/fork noise

Phase 2 — Data cleanup and classification:
- Schema: 6 new curation columns + 4 indexes
- curate.mjs: 8-step pipeline (classify, dedup, fork detection, etc.)
- Result: 197K → 60K unique → 16K browse-ready skills
- Bug fix: securityStatus was computed but never stored during crawl

Phase 3 — UI browse-ready filters:
- browseReadyFilter applied to 17+ query functions
- Homepage stats show accurate browse-ready counts
- Stats API filtered (previously had no WHERE clause)
- Category counts recalculated (e.g. 45K → 3.1K)
- Featured skills exclude duplicates and aggregators

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
airano
2026-02-19 14:31:25 +03:30
parent 31f5df0900
commit caca09fbe7
12 changed files with 2615 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
apps/web/app/[locale]/page.tsx
View File

@@ -16,27 +16,32 @@ async function getStats() {
try {
const db = createDb();
// Get total skills count (SKILL.md only - real reusable skills)
// Browse-ready filter: exclude duplicates and aggregators
const browseReady = sql`${skills.isDuplicate} = false AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`;
// Get total skills count (browse-ready, SKILL.md only)
const skillsResult = await db
.select({ count: sql<number>`count(*)::int` })
.from(skills)
.where(sql`${skills.sourceFormat} = 'skill.md' AND ${skills.isBlocked} = false`);
.where(sql`${skills.sourceFormat} = 'skill.md' AND ${skills.isBlocked} = false AND ${browseReady}`);
const totalSkills = skillsResult[0]?.count ?? 0;
// Get total downloads
// Get total downloads (browse-ready only)
const downloadsResult = await db
.select({ sum: sql<number>`coalesce(sum(${skills.downloadCount}), 0)::int` })
.from(skills);
.from(skills)
.where(browseReady);
const totalDownloads = downloadsResult[0]?.sum ?? 0;
// Get total categories
const categories = await categoryQueries.getAll(db);
const totalCategories = categories.length;
// Get unique contributors (github owners)
// Get unique contributors (browse-ready skills only)
const contributorsResult = await db
.select({ count: sql<number>`count(distinct ${skills.githubOwner})::int` })
.from(skills);
.from(skills)
.where(browseReady);
const totalContributors = contributorsResult[0]?.count ?? 0;
return {

21
apps/web/app/api/stats/route.test.ts
View File

@@ -18,17 +18,26 @@ vi.mock('@/lib/cache', () => ({
// Mock the db module - must be before imports that use it
vi.mock('@skillhub/db', () => {
const mockStatsRow = [{ totalSkills: 100, totalDownloads: 5000, totalContributors: 50 }];
const mockCategoryRow = [{ count: 8 }];
return {
createDb: vi.fn(() => ({
select: vi.fn().mockReturnValue({
from: vi.fn().mockResolvedValue([
{ totalSkills: 100, totalDownloads: 5000, totalContributors: 50 },
]),
from: vi.fn((table: unknown) => {
// categories table query returns count directly (no .where())
if (table === 'categories-table') {
return Promise.resolve(mockCategoryRow);
}
// skills table query has .where() chained
return {
where: vi.fn().mockResolvedValue(mockStatsRow),
};
}),
}),
})),
skills: { downloadCount: 'download_count', githubOwner: 'github_owner' },
categories: {},
sql: vi.fn(() => 'mock-sql'),
skills: { downloadCount: 'download_count', githubOwner: 'github_owner', isDuplicate: 'is_duplicate', skillType: 'skill_type' },
categories: 'categories-table',
sql: vi.fn((..._args: unknown[]) => 'mock-sql'),
};
});

8
apps/web/app/api/stats/route.ts
View File

@@ -34,14 +34,18 @@ export async function GET(request: NextRequest) {
});
}
// Consolidate all skill stats into a single query for better performance
// Browse-ready filter: exclude duplicates and aggregators
const browseReady = sql`${skills.isDuplicate} = false AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`;
// Consolidate all skill stats into a single query (browse-ready only)
const statsResult = await db
.select({
totalSkills: sql<number>`count(*)::int`,
totalDownloads: sql<number>`coalesce(sum(${skills.downloadCount}), 0)::int`,
totalContributors: sql<number>`count(distinct ${skills.githubOwner})::int`,
})
.from(skills);
.from(skills)
.where(browseReady);
// Get category count in separate query (different table)
const categoryResult = await db

42
packages/db/src/queries.ts
View File

@@ -30,6 +30,13 @@ function getRawClientLocal() {
type DB = PostgresJsDatabase<typeof schema>;
/**
* Curation filter: excludes duplicates and aggregator-only skills.
* Skills with skill_type=NULL (newly crawled, not yet curated) are included
* so they don't disappear between curate.mjs runs.
*/
const browseReadyFilter = sql`(${skills.isDuplicate} = false) AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`;
/**
* Skill queries
*/
@@ -76,7 +83,8 @@ export const skillQueries = {
} = options;
const conditions = [
eq(skills.isBlocked, false), // Filter out blocked skills
eq(skills.isBlocked, false),
browseReadyFilter,
];
// Filter by source format (default: SKILL.md only; 'all' shows everything)
@@ -200,7 +208,8 @@ export const skillQueries = {
const { query, category, platform, sourceFormat = 'skill.md', minStars = 0, minSecurity, verified } = options;
const conditions = [
eq(skills.isBlocked, false), // Filter out blocked skills
eq(skills.isBlocked, false),
browseReadyFilter,
];
// Filter by source format (default: SKILL.md only; 'all' shows everything)
@@ -277,7 +286,7 @@ export const skillQueries = {
return db
.select()
.from(skills)
.where(and(eq(skills.isFeatured, true), eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md')))
.where(and(eq(skills.isFeatured, true), eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'), browseReadyFilter))
.orderBy(desc(skills.githubStars))
.limit(limit)
.offset(offset);
@@ -290,7 +299,7 @@ export const skillQueries = {
const result = await db
.select({ count: sql<number>`cast(count(*) as int)` })
.from(skills)
.where(and(eq(skills.isFeatured, true), eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md')));
.where(and(eq(skills.isFeatured, true), eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'), browseReadyFilter));
return result[0]?.count ?? 0;
},
@@ -298,14 +307,14 @@ export const skillQueries = {
* Get trending skills (most downloads in recent period)
*/
getTrending: async (db: DB, limit = 10) => {
return db.select().from(skills).where(and(eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'))).orderBy(desc(skills.downloadCount)).limit(limit);
return db.select().from(skills).where(and(eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'), browseReadyFilter)).orderBy(desc(skills.downloadCount)).limit(limit);
},
/**
* Get recently added skills with pagination (sorted by creation date)
*/
getRecent: async (db: DB, limit = 10, offset = 0) => {
return db.select().from(skills).where(and(eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'))).orderBy(desc(skills.createdAt)).limit(limit).offset(offset);
return db.select().from(skills).where(and(eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'), browseReadyFilter)).orderBy(desc(skills.createdAt)).limit(limit).offset(offset);
},
/**
@@ -320,6 +329,7 @@ export const skillQueries = {
and(
eq(skills.isBlocked, false),
eq(skills.sourceFormat, 'skill.md'),
browseReadyFilter,
gte(skills.createdAt, sevenDaysAgo)
)
)
@@ -340,6 +350,7 @@ export const skillQueries = {
and(
eq(skills.isBlocked, false),
eq(skills.sourceFormat, 'skill.md'),
browseReadyFilter,
gte(skills.createdAt, sevenDaysAgo)
)
);
@@ -358,6 +369,7 @@ export const skillQueries = {
and(
eq(skills.isBlocked, false),
eq(skills.sourceFormat, 'skill.md'),
browseReadyFilter,
sql`${skills.createdAt} < ${sevenDaysAgo}`,
gte(skills.updatedAt, sevenDaysAgo),
sql`${skills.updatedAt} > ${skills.createdAt} + interval '1 hour'`
@@ -380,6 +392,7 @@ export const skillQueries = {
and(
eq(skills.isBlocked, false),
eq(skills.sourceFormat, 'skill.md'),
browseReadyFilter,
sql`${skills.createdAt} < ${sevenDaysAgo}`,
gte(skills.updatedAt, sevenDaysAgo),
sql`${skills.updatedAt} > ${skills.createdAt} + interval '1 hour'`
@@ -395,7 +408,7 @@ export const skillQueries = {
const result = await db
.select({ count: sql<number>`cast(count(*) as int)` })
.from(skills)
.where(and(eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md')));
.where(and(eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'), browseReadyFilter));
return result[0]?.count ?? 0;
},
@@ -410,7 +423,7 @@ export const skillQueries = {
githubOwner: skills.githubOwner,
})
.from(skills)
.where(and(eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md')));
.where(and(eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'), browseReadyFilter));
},
/**
@@ -467,7 +480,7 @@ export const skillQueries = {
return db
.select()
.from(skills)
.where(and(eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md')))
.where(and(eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'), browseReadyFilter))
.orderBy(
desc(
sql`(
@@ -580,6 +593,7 @@ export const skillQueries = {
githubStars: skill.githubStars,
githubForks: skill.githubForks,
securityScore: skill.securityScore,
securityStatus: skill.securityStatus,
sourceFormat: skill.sourceFormat,
contentHash: skill.contentHash,
rawContent: skill.rawContent,
@@ -751,7 +765,7 @@ export const skillQueries = {
) => {
const { limit = 24, offset = 0, sortBy = 'popularity', repo } = options;
const conditions = [eq(skills.githubOwner, owner), eq(skills.isBlocked, false)];
const conditions = [eq(skills.githubOwner, owner), eq(skills.isBlocked, false), browseReadyFilter];
if (repo) conditions.push(eq(skills.githubRepo, repo));
const ownerCondition = and(...conditions);
@@ -828,7 +842,7 @@ export const skillQueries = {
skillCount: sql<number>`count(*) OVER (PARTITION BY ${skills.githubRepo})::int`,
})
.from(skills)
.where(and(eq(skills.githubOwner, owner), eq(skills.isBlocked, false)))
.where(and(eq(skills.githubOwner, owner), eq(skills.isBlocked, false), browseReadyFilter))
.orderBy(skills.githubRepo, desc(skills.githubStars));
},
@@ -836,7 +850,7 @@ export const skillQueries = {
* Count skills by owner (for pagination), optionally filtered by repo
*/
countByOwner: async (db: DB, owner: string, repo?: string): Promise<number> => {
const conditions = [eq(skills.githubOwner, owner), eq(skills.isBlocked, false)];
const conditions = [eq(skills.githubOwner, owner), eq(skills.isBlocked, false), browseReadyFilter];
if (repo) conditions.push(eq(skills.githubRepo, repo));
const result = await db
.select({ count: sql<number>`cast(count(*) as int)` })
@@ -858,7 +872,7 @@ export const skillQueries = {
maxStars: sql<number>`COALESCE(MAX(${skills.githubStars}), 0)::int`,
})
.from(skills)
.where(and(eq(skills.githubOwner, owner), eq(skills.isBlocked, false)));
.where(and(eq(skills.githubOwner, owner), eq(skills.isBlocked, false), browseReadyFilter));
return result[0] ?? { totalSkills: 0, totalDownloads: 0, totalViews: 0, totalRepos: 0, maxStars: 0 };
},
@@ -891,7 +905,7 @@ export const categoryQueries = {
.select({ skill: skills })
.from(skillCategories)
.innerJoin(skills, eq(skillCategories.skillId, skills.id))
.where(eq(skillCategories.categoryId, categoryId))
.where(and(eq(skillCategories.categoryId, categoryId), sql`${browseReadyFilter}`))
.orderBy(desc(skills.githubStars))
.limit(limit)
.offset(offset);

17
packages/db/src/schema.ts
View File

@@ -67,6 +67,19 @@ export const skills = pgTable(
isBlocked: boolean('is_blocked').default(false), // Blocked from re-indexing (owner requested removal)
lastScanned: timestamp('last_scanned'),
// Curation (populated by batch scripts, not crawler)
qualityScore: integer('quality_score'), // 0-100 from analyzer
qualityDetails: jsonb('quality_details').$type<{
documentation: number;
maintenance: number;
popularity: number;
factors: Array<{ name: string; score: number; weight: number; details?: string }>;
}>(),
skillType: text('skill_type').$type<'standalone' | 'project-bound' | 'collection' | 'aggregator'>(),
isDuplicate: boolean('is_duplicate').default(false),
canonicalSkillId: text('canonical_skill_id'), // points to the "original" if this is a duplicate
repoSkillCount: integer('repo_skill_count'), // cached count of skills in same repo
// Content (cached)
contentHash: text('content_hash'),
rawContent: text('raw_content'),
@@ -103,6 +116,10 @@ export const skills = pgTable(
updatedIdx: index('idx_skills_updated').on(table.updatedAt),
sourceFormatIdx: index('idx_skills_source_format').on(table.sourceFormat),
lastDownloadedIdx: index('idx_skills_last_downloaded').on(table.lastDownloadedAt),
qualityIdx: index('idx_skills_quality').on(table.qualityScore),
skillTypeIdx: index('idx_skills_type').on(table.skillType),
duplicateIdx: index('idx_skills_duplicate').on(table.isDuplicate),
contentHashIdx: index('idx_skills_content_hash').on(table.contentHash),
})
);

505
scripts/curation/curate.mjs Normal file
View File

@@ -0,0 +1,505 @@
#!/usr/bin/env node
/**
* Phase 2: Data Cleanup & Classification
*
* Runs all curation steps in order:
* Step 1: Compute repo_skill_count for every skill
* Step 2: Mark aggregators (repos with 50+ skills)
* Step 3: Classify remaining repos (collection / standalone / project-bound)
* Step 4: Fill missing content_hash values
* Step 5: Mark duplicates by content_hash (canonical = highest stars or oldest)
* Step 6: Detect fork marketplace repos
* Step 7: Recalculate category skill counts (browse-ready only)
* Step 8: Summary report
*
* Usage:
* DATABASE_URL=postgres://... node scripts/curation/curate.mjs
* # Or with local Docker:
* DATABASE_URL=postgresql://postgres:postgres@localhost:5432/skillhub node scripts/curation/curate.mjs
*
* Options:
* --dry-run Show what would change without writing
* --step=N Run only step N (1-8)
*/
import { createRequire } from 'module';
import { resolve, dirname } from 'path';
import { fileURLToPath } from 'url';
const __dirname = dirname(fileURLToPath(import.meta.url));
// ─── Find pg module ───
let pg;
const tryPaths = [
resolve(__dirname, '../..', 'package.json'),
'/tmp/package.json',
process.env.APPDATA ? resolve(process.env.APPDATA, '..', 'Local', 'Temp', 'package.json') : null,
].filter(Boolean);
for (const p of tryPaths) {
try { pg = createRequire(p)('pg'); break; } catch {}
}
if (!pg) { console.error('pg not found. Run: npm install pg'); process.exit(1); }
// ─── Config ───
const DATABASE_URL = process.env.DATABASE_URL || 'postgresql://postgres:postgres@localhost:5432/skillhub';
const DRY_RUN = process.argv.includes('--dry-run');
const stepArg = process.argv.find(a => a.startsWith('--step='));
const ONLY_STEP = stepArg ? parseInt(stepArg.split('=')[1]) : null;
// ─── Helpers ───
function header(step, title) {
console.log(`\n${'='.repeat(70)}\n STEP ${step}: ${title}\n${'='.repeat(70)}`);
}
const n = v => Number(v ?? 0).toLocaleString('en-US');
// ─── Database ───
let client;
async function connect() {
client = new pg.Client({
connectionString: DATABASE_URL,
ssl: false,
connectionTimeoutMillis: 15000,
query_timeout: 600000, // 10 min for big updates
keepAlive: true,
});
await client.connect();
console.log('Connected to database');
}
async function query(sql, params = []) {
const result = await client.query(sql, params);
return result;
}
// ─── Step 1: Compute repo_skill_count ───
async function step1_repoSkillCount() {
header(1, 'COMPUTE repo_skill_count');
const countResult = await query(`
SELECT COUNT(*)::int AS total
FROM skills
WHERE is_blocked = false AND repo_skill_count IS NULL
`);
console.log(` Skills without repo_skill_count: ${n(countResult.rows[0].total)}`);
if (DRY_RUN) { console.log(' [DRY RUN] Would update all skills'); return; }
const result = await query(`
UPDATE skills s SET repo_skill_count = sub.cnt
FROM (
SELECT github_owner, github_repo, COUNT(*)::int AS cnt
FROM skills WHERE is_blocked = false
GROUP BY github_owner, github_repo
) sub
WHERE s.github_owner = sub.github_owner
AND s.github_repo = sub.github_repo
AND s.is_blocked = false
`);
console.log(` Updated: ${n(result.rowCount)} skills`);
}
// ─── Step 2: Mark aggregators ───
async function step2_markAggregators() {
header(2, 'MARK AGGREGATORS (repos with 50+ skills)');
// Preview
const preview = await query(`
SELECT github_owner || '/' || github_repo AS repo, COUNT(*)::int AS skills,
MAX(github_stars)::int AS stars
FROM skills WHERE is_blocked = false
GROUP BY github_owner, github_repo
HAVING COUNT(*) >= 50
ORDER BY skills DESC LIMIT 20
`);
console.log(` Top aggregator repos (${preview.rowCount} total):`);
for (const r of preview.rows.slice(0, 10)) {
console.log(` ${r.repo}: ${n(r.skills)} skills (${n(r.stars)} stars)`);
}
if (DRY_RUN) { console.log(' [DRY RUN] Would mark skills in these repos'); return; }
const result = await query(`
UPDATE skills s SET skill_type = 'aggregator'
FROM (
SELECT github_owner, github_repo
FROM skills WHERE is_blocked = false
GROUP BY github_owner, github_repo
HAVING COUNT(*) >= 50
) agg
WHERE s.github_owner = agg.github_owner
AND s.github_repo = agg.github_repo
AND s.is_blocked = false
AND s.skill_type IS NULL
`);
console.log(` Marked as aggregator: ${n(result.rowCount)} skills`);
}
// ─── Step 3: Classify remaining repos ───
async function step3_classifyRemaining() {
header(3, 'CLASSIFY REMAINING REPOS');
// 3a: repos with 10-49 skills, name contains marketplace/awesome/collection → aggregator
if (!DRY_RUN) {
const aggByName = await query(`
UPDATE skills s SET skill_type = 'aggregator'
FROM (
SELECT github_owner, github_repo
FROM skills WHERE is_blocked = false AND skill_type IS NULL
GROUP BY github_owner, github_repo
HAVING COUNT(*) >= 10
AND (github_repo ILIKE '%marketplace%'
OR github_repo ILIKE '%awesome%'
OR github_repo ILIKE '%collection%'
OR github_repo ILIKE '%registry%')
) agg
WHERE s.github_owner = agg.github_owner
AND s.github_repo = agg.github_repo
AND s.is_blocked = false
AND s.skill_type IS NULL
`);
console.log(` Aggregator by name (10+ & marketplace/awesome/registry): ${n(aggByName.rowCount)}`);
}
// 3b: repos with 3-49 skills → collection
if (!DRY_RUN) {
const collections = await query(`
UPDATE skills s SET skill_type = 'collection'
FROM (
SELECT github_owner, github_repo
FROM skills WHERE is_blocked = false AND skill_type IS NULL
GROUP BY github_owner, github_repo
HAVING COUNT(*) >= 3
) col
WHERE s.github_owner = col.github_owner
AND s.github_repo = col.github_repo
AND s.is_blocked = false
AND s.skill_type IS NULL
`);
console.log(` Collection (3+ skills in repo): ${n(collections.rowCount)}`);
}
// 3c: single/two-skill repos with project-bound name patterns → project-bound
if (!DRY_RUN) {
const projectBound = await query(`
UPDATE skills SET skill_type = 'project-bound'
WHERE is_blocked = false AND skill_type IS NULL
AND repo_skill_count <= 2
AND (name ILIKE '%my-%' OR name ILIKE '%my\\_%' ESCAPE '\\'
OR name ILIKE '%project%' OR name ILIKE '%team%' OR name ILIKE '%internal%'
OR name ILIKE '%.mdc' OR name ILIKE '%cursorrule%'
OR name ILIKE '%config%' OR name ILIKE '%setup%')
`);
console.log(` Project-bound (name pattern): ${n(projectBound.rowCount)}`);
}
// 3d: remaining → standalone
if (!DRY_RUN) {
const standalone = await query(`
UPDATE skills SET skill_type = 'standalone'
WHERE is_blocked = false AND skill_type IS NULL
`);
console.log(` Standalone (remaining): ${n(standalone.rowCount)}`);
}
// Summary
const summary = await query(`
SELECT skill_type, COUNT(*)::int AS count
FROM skills WHERE is_blocked = false
GROUP BY skill_type ORDER BY count DESC
`);
console.log('\n Classification summary:');
for (const r of summary.rows) {
console.log(` ${(r.skill_type || 'null').padEnd(15)} ${n(r.count)}`);
}
}
// ─── Step 4: Fill missing content_hash ───
async function step4_fillContentHash() {
header(4, 'FILL MISSING content_hash');
const countResult = await query(`
SELECT COUNT(*)::int AS total
FROM skills WHERE is_blocked = false AND content_hash IS NULL AND raw_content IS NOT NULL
`);
const missing = countResult.rows[0].total;
console.log(` Skills missing content_hash: ${n(missing)}`);
if (missing === 0) { console.log(' Nothing to do'); return; }
if (DRY_RUN) { console.log(' [DRY RUN] Would compute hash for these skills'); return; }
// Use md5 as a reliable hash function available in PostgreSQL
const result = await query(`
UPDATE skills SET content_hash = md5(raw_content)
WHERE is_blocked = false AND content_hash IS NULL AND raw_content IS NOT NULL
`);
console.log(` Computed content_hash (md5) for: ${n(result.rowCount)} skills`);
// Note: existing hashes use a JS numeric hash (analyzer.ts hashContent).
// We now use md5 for missing ones. For dedup purposes, same content → same hash
// regardless of algorithm, since we compare within hash groups.
// But mixed algorithms mean same content could have different hashes.
// Solution: re-hash ALL with md5 for consistency.
console.log(' Re-hashing ALL skills with md5 for consistency...');
const rehash = await query(`
UPDATE skills SET content_hash = md5(raw_content)
WHERE is_blocked = false AND raw_content IS NOT NULL
`);
console.log(` Re-hashed: ${n(rehash.rowCount)} skills`);
}
// ─── Step 5: Mark duplicates ───
async function step5_markDuplicates() {
header(5, 'MARK DUPLICATES BY content_hash');
// Find duplicate groups
const dupGroups = await query(`
SELECT content_hash, COUNT(*)::int AS copies
FROM skills
WHERE is_blocked = false AND content_hash IS NOT NULL AND is_duplicate = false
GROUP BY content_hash
HAVING COUNT(*) > 1
ORDER BY copies DESC
`);
const totalDupGroups = dupGroups.rowCount;
const totalDupSkills = dupGroups.rows.reduce((sum, r) => sum + r.copies, 0);
const removable = dupGroups.rows.reduce((sum, r) => sum + r.copies - 1, 0);
console.log(` Duplicate groups: ${n(totalDupGroups)}`);
console.log(` Total skills in dup groups: ${n(totalDupSkills)}`);
console.log(` Removable (keeping 1 per group): ${n(removable)}`);
console.log(` Top 5 by copies:`);
for (const r of dupGroups.rows.slice(0, 5)) {
console.log(` hash=${r.content_hash}: ${r.copies} copies`);
}
if (DRY_RUN) { console.log(' [DRY RUN] Would mark duplicates'); return; }
// For each duplicate group:
// - canonical = highest github_stars, then oldest created_at
// - rest = is_duplicate = true, canonical_skill_id = canonical.id
const markResult = await query(`
WITH ranked AS (
SELECT id, content_hash,
ROW_NUMBER() OVER (
PARTITION BY content_hash
ORDER BY github_stars DESC NULLS LAST,
created_at ASC
) AS rn
FROM skills
WHERE is_blocked = false AND content_hash IS NOT NULL AND is_duplicate = false
),
canonicals AS (
SELECT content_hash, id AS canonical_id
FROM ranked WHERE rn = 1
)
UPDATE skills s
SET is_duplicate = true,
canonical_skill_id = c.canonical_id
FROM ranked r
JOIN canonicals c ON r.content_hash = c.content_hash
WHERE s.id = r.id
AND r.rn > 1
`);
console.log(` Marked as duplicate: ${n(markResult.rowCount)} skills`);
// Verify
const verify = await query(`
SELECT
COUNT(*) FILTER (WHERE is_duplicate = false)::int AS unique_skills,
COUNT(*) FILTER (WHERE is_duplicate = true)::int AS duplicates
FROM skills WHERE is_blocked = false
`);
console.log(` After dedup: ${n(verify.rows[0].unique_skills)} unique, ${n(verify.rows[0].duplicates)} duplicates`);
}
// ─── Step 6: Detect fork marketplace repos ───
async function step6_detectForks() {
header(6, 'DETECT FORK MARKETPLACE REPOS');
// Known fork patterns: repos with same name across many owners
const forkPatterns = await query(`
SELECT github_repo, COUNT(DISTINCT github_owner)::int AS owners,
COUNT(*)::int AS total_skills
FROM skills
WHERE is_blocked = false
AND repo_skill_count >= 20
GROUP BY github_repo
HAVING COUNT(DISTINCT github_owner) >= 3
ORDER BY owners DESC
`);
console.log(` Repo names appearing in 3+ owners (with 20+ skills each):`);
for (const r of forkPatterns.rows) {
console.log(` ${r.github_repo}: ${r.owners} owners, ${n(r.total_skills)} skills`);
}
if (DRY_RUN) { console.log(' [DRY RUN] Would mark fork repo skills'); return; }
// For fork repos, all copies should be checked as duplicates
// The canonical is already handled by step 5 (content_hash dedup)
// Here we just ensure they're typed correctly as aggregator
if (forkPatterns.rows.length > 0) {
const repoNames = forkPatterns.rows.map(r => r.github_repo);
const placeholders = repoNames.map((_, i) => `$${i + 1}`).join(',');
const markForks = await query(`
UPDATE skills SET skill_type = 'aggregator'
WHERE is_blocked = false
AND github_repo IN (${placeholders})
AND repo_skill_count >= 20
AND (skill_type IS NULL OR skill_type != 'aggregator')
`, repoNames);
console.log(` Re-classified as aggregator (fork repos): ${n(markForks.rowCount)}`);
}
}
// ─── Step 7: Recalculate category counts ───
async function step7_categoryCounts() {
header(7, 'RECALCULATE CATEGORY COUNTS');
// Update skill_count on each category to only count browse-ready skills
const result = await query(`
UPDATE categories c
SET skill_count = sub.cnt
FROM (
SELECT sc.category_id, COUNT(*)::int AS cnt
FROM skill_categories sc
JOIN skills s ON sc.skill_id = s.id
WHERE s.is_blocked = false
AND s.is_duplicate = false
AND (s.skill_type IS NULL OR s.skill_type != 'aggregator')
GROUP BY sc.category_id
) sub
WHERE c.id = sub.category_id
AND c.skill_count IS DISTINCT FROM sub.cnt
`);
console.log(` Updated ${result.rowCount} category counts (browse-ready only)`);
// Also zero out categories that have no browse-ready skills
const zeroResult = await query(`
UPDATE categories c
SET skill_count = 0
WHERE c.skill_count > 0
AND NOT EXISTS (
SELECT 1 FROM skill_categories sc
JOIN skills s ON sc.skill_id = s.id
WHERE sc.category_id = c.id
AND s.is_blocked = false
AND s.is_duplicate = false
AND (s.skill_type IS NULL OR s.skill_type != 'aggregator')
)
`);
if (zeroResult.rowCount > 0) {
console.log(` Zeroed ${zeroResult.rowCount} categories with no browse-ready skills`);
}
// Show category counts
const cats = await query(`
SELECT name, skill_count FROM categories
WHERE id NOT LIKE 'parent-%'
ORDER BY skill_count DESC
LIMIT 10
`);
console.log(' Top 10 categories by browse-ready count:');
for (const r of cats.rows) {
console.log(` ${n(r.skill_count).padStart(6)} ${r.name}`);
}
}
// ─── Step 8: Summary ───
async function step8_summary() {
header(8, 'FINAL SUMMARY');
const summary = await query(`
SELECT
COUNT(*)::int AS total,
COUNT(*) FILTER (WHERE is_duplicate = false)::int AS unique_skills,
COUNT(*) FILTER (WHERE is_duplicate = true)::int AS duplicates,
COUNT(*) FILTER (WHERE skill_type = 'standalone' AND is_duplicate = false)::int AS standalone,
COUNT(*) FILTER (WHERE skill_type = 'collection' AND is_duplicate = false)::int AS collection,
COUNT(*) FILTER (WHERE skill_type = 'aggregator' AND is_duplicate = false)::int AS aggregator,
COUNT(*) FILTER (WHERE skill_type = 'project-bound' AND is_duplicate = false)::int AS project_bound,
COUNT(*) FILTER (WHERE skill_type IS NULL AND is_duplicate = false)::int AS unclassified
FROM skills WHERE is_blocked = false
`);
const s = summary.rows[0];
console.log(`
┌─────────────────────────────────────────────────────┐
│ CURATION SUMMARY │
├─────────────────────────────────────────────────────┤
│ Total skills: ${n(s.total).padStart(8)} │
│ Unique skills: ${n(s.unique_skills).padStart(8)} │
│ Duplicates: ${n(s.duplicates).padStart(8)} │
├─────────────────────────────────────────────────────┤
│ Unique by type: │
│ Standalone: ${n(s.standalone).padStart(8)} │
│ Collection: ${n(s.collection).padStart(8)} │
│ Aggregator: ${n(s.aggregator).padStart(8)} │
│ Project-bound: ${n(s.project_bound).padStart(8)} │
│ Unclassified: ${n(s.unclassified).padStart(8)} │
└─────────────────────────────────────────────────────┘
`);
// Interesting standalone skills
const topStandalone = await query(`
SELECT id, name, github_stars AS stars, COALESCE(download_count,0) AS dl,
LEFT(description, 70) AS description
FROM skills
WHERE is_blocked = false AND is_duplicate = false
AND skill_type = 'standalone'
ORDER BY github_stars DESC NULLS LAST
LIMIT 20
`);
console.log(' Top 20 standalone skills by stars:');
for (const r of topStandalone.rows) {
console.log(` [${n(r.stars)}★ ${n(r.dl)}↓] ${r.id}`);
console.log(` ${r.description}`);
}
// Browse-ready count (what users would see)
const browseReady = await query(`
SELECT COUNT(*)::int AS count
FROM skills
WHERE is_blocked = false
AND is_duplicate = false
AND skill_type IN ('standalone', 'collection')
`);
console.log(`\n Browse-ready skills (standalone + collection, unique): ${n(browseReady.rows[0].count)}`);
}
// ─── Main ───
async function main() {
const t0 = Date.now();
await connect();
if (DRY_RUN) console.log('\n *** DRY RUN MODE — no changes will be made ***\n');
const steps = [
step1_repoSkillCount,
step2_markAggregators,
step3_classifyRemaining,
step4_fillContentHash,
step5_markDuplicates,
step6_detectForks,
step7_categoryCounts,
step8_summary,
];
for (let i = 0; i < steps.length; i++) {
if (ONLY_STEP && ONLY_STEP !== i + 1) continue;
await steps[i]();
}
const dur = ((Date.now() - t0) / 1000).toFixed(1);
console.log(`\nCompleted in ${dur}s`);
await client.end().catch(() => {});
}
main().catch(async e => {
console.error('FAILED:', e.message || e);
await client?.end().catch(() => {});
process.exit(1);
});

214
scripts/curation/curate.sql Normal file
View File

@@ -0,0 +1,214 @@
-- ============================================================
-- Phase 2: Data Cleanup & Classification (Pure SQL version)
-- Run inside DB container:
-- psql -U postgres -d skillhub -f /tmp/curate.sql
--
-- Safe to run multiple times (idempotent).
-- ============================================================
\echo '======================================================================'
\echo ' STEP 1: COMPUTE repo_skill_count'
\echo '======================================================================'
UPDATE skills s SET repo_skill_count = sub.cnt
FROM (
SELECT github_owner, github_repo, COUNT(*)::int AS cnt
FROM skills WHERE is_blocked = false
GROUP BY github_owner, github_repo
) sub
WHERE s.github_owner = sub.github_owner
AND s.github_repo = sub.github_repo
AND s.is_blocked = false;
\echo ' Step 1 done. Checking counts:'
SELECT 'repo_skill_count set' AS step,
COUNT(*) FILTER (WHERE repo_skill_count IS NOT NULL) AS done,
COUNT(*) FILTER (WHERE repo_skill_count IS NULL) AS remaining
FROM skills WHERE is_blocked = false;
\echo ''
\echo '======================================================================'
\echo ' STEP 2: MARK AGGREGATORS (repos with 50+ skills)'
\echo '======================================================================'
UPDATE skills s SET skill_type = 'aggregator'
FROM (
SELECT github_owner, github_repo
FROM skills WHERE is_blocked = false
GROUP BY github_owner, github_repo
HAVING COUNT(*) >= 50
) agg
WHERE s.github_owner = agg.github_owner
AND s.github_repo = agg.github_repo
AND s.is_blocked = false
AND s.skill_type IS NULL;
\echo ' Step 2 done.'
\echo ''
\echo '======================================================================'
\echo ' STEP 3: CLASSIFY REMAINING REPOS'
\echo '======================================================================'
-- 3a: 10-49 skills + name contains marketplace/awesome/collection/registry → aggregator
UPDATE skills s SET skill_type = 'aggregator'
FROM (
SELECT github_owner, github_repo
FROM skills WHERE is_blocked = false AND skill_type IS NULL
GROUP BY github_owner, github_repo
HAVING COUNT(*) >= 10
AND (github_repo ILIKE '%marketplace%'
OR github_repo ILIKE '%awesome%'
OR github_repo ILIKE '%collection%'
OR github_repo ILIKE '%registry%')
) agg
WHERE s.github_owner = agg.github_owner
AND s.github_repo = agg.github_repo
AND s.is_blocked = false
AND s.skill_type IS NULL;
-- 3b: 3+ skills → collection
UPDATE skills s SET skill_type = 'collection'
FROM (
SELECT github_owner, github_repo
FROM skills WHERE is_blocked = false AND skill_type IS NULL
GROUP BY github_owner, github_repo
HAVING COUNT(*) >= 3
) col
WHERE s.github_owner = col.github_owner
AND s.github_repo = col.github_repo
AND s.is_blocked = false
AND s.skill_type IS NULL;
-- 3c: project-bound name patterns
UPDATE skills SET skill_type = 'project-bound'
WHERE is_blocked = false AND skill_type IS NULL
AND repo_skill_count <= 2
AND (name ILIKE '%my-%' OR name ILIKE '%my\_%' ESCAPE '\'
OR name ILIKE '%project%' OR name ILIKE '%team%' OR name ILIKE '%internal%'
OR name ILIKE '%.mdc' OR name ILIKE '%cursorrule%'
OR name ILIKE '%config%' OR name ILIKE '%setup%');
-- 3d: remaining → standalone
UPDATE skills SET skill_type = 'standalone'
WHERE is_blocked = false AND skill_type IS NULL;
\echo ' Step 3 done. Classification:'
SELECT skill_type, COUNT(*)::int AS count
FROM skills WHERE is_blocked = false
GROUP BY skill_type ORDER BY count DESC;
\echo ''
\echo '======================================================================'
\echo ' STEP 4: FILL content_hash WITH md5 (for consistency)'
\echo '======================================================================'
-- Re-hash ALL with md5 for consistent dedup
UPDATE skills SET content_hash = md5(raw_content)
WHERE is_blocked = false AND raw_content IS NOT NULL;
\echo ' Step 4 done.'
SELECT 'content_hash' AS step,
COUNT(*) FILTER (WHERE content_hash IS NOT NULL) AS has_hash,
COUNT(*) FILTER (WHERE content_hash IS NULL) AS no_hash
FROM skills WHERE is_blocked = false;
\echo ''
\echo '======================================================================'
\echo ' STEP 5: MARK DUPLICATES BY content_hash'
\echo '======================================================================'
-- First reset any previous duplicate marks (for idempotency)
UPDATE skills SET is_duplicate = false, canonical_skill_id = NULL
WHERE is_blocked = false AND is_duplicate = true;
-- Mark duplicates: keep the one with most stars (or oldest) as canonical
WITH ranked AS (
SELECT id, content_hash,
ROW_NUMBER() OVER (
PARTITION BY content_hash
ORDER BY github_stars DESC NULLS LAST,
created_at ASC
) AS rn
FROM skills
WHERE is_blocked = false AND content_hash IS NOT NULL
),
canonicals AS (
SELECT content_hash, id AS canonical_id
FROM ranked WHERE rn = 1
)
UPDATE skills s
SET is_duplicate = true,
canonical_skill_id = c.canonical_id
FROM ranked r
JOIN canonicals c ON r.content_hash = c.content_hash
WHERE s.id = r.id
AND r.rn > 1;
\echo ' Step 5 done.'
SELECT 'dedup' AS step,
COUNT(*) FILTER (WHERE is_duplicate = false) AS unique_skills,
COUNT(*) FILTER (WHERE is_duplicate = true) AS duplicates
FROM skills WHERE is_blocked = false;
\echo ''
\echo '======================================================================'
\echo ' STEP 6: DETECT FORK MARKETPLACE REPOS'
\echo '======================================================================'
-- Re-classify fork repos (same repo name in 3+ owners with 20+ skills) as aggregator
UPDATE skills SET skill_type = 'aggregator'
WHERE is_blocked = false
AND repo_skill_count >= 20
AND (skill_type IS NULL OR skill_type != 'aggregator')
AND github_repo IN (
SELECT github_repo
FROM skills
WHERE is_blocked = false AND repo_skill_count >= 20
GROUP BY github_repo
HAVING COUNT(DISTINCT github_owner) >= 3
);
\echo ' Step 6 done. Fork patterns:'
SELECT github_repo, COUNT(DISTINCT github_owner)::int AS owners,
COUNT(*)::int AS total_skills
FROM skills
WHERE is_blocked = false AND repo_skill_count >= 20
GROUP BY github_repo
HAVING COUNT(DISTINCT github_owner) >= 3
ORDER BY owners DESC
LIMIT 15;
\echo ''
\echo '======================================================================'
\echo ' STEP 7: FINAL SUMMARY'
\echo '======================================================================'
SELECT
COUNT(*)::int AS total,
COUNT(*) FILTER (WHERE is_duplicate = false)::int AS unique_skills,
COUNT(*) FILTER (WHERE is_duplicate = true)::int AS duplicates,
COUNT(*) FILTER (WHERE skill_type = 'standalone' AND is_duplicate = false)::int AS standalone,
COUNT(*) FILTER (WHERE skill_type = 'collection' AND is_duplicate = false)::int AS collection_type,
COUNT(*) FILTER (WHERE skill_type = 'aggregator' AND is_duplicate = false)::int AS aggregator,
COUNT(*) FILTER (WHERE skill_type = 'project-bound' AND is_duplicate = false)::int AS project_bound,
COUNT(*) FILTER (WHERE skill_type IN ('standalone','collection') AND is_duplicate = false)::int AS browse_ready
FROM skills WHERE is_blocked = false;
\echo ''
\echo ' Top 20 standalone skills by stars:'
SELECT id, github_stars AS stars, COALESCE(download_count,0) AS dl,
LEFT(description, 70) AS description
FROM skills
WHERE is_blocked = false AND is_duplicate = false AND skill_type = 'standalone'
ORDER BY github_stars DESC NULLS LAST
LIMIT 20;
\echo ''
\echo ' DONE!'

652
scripts/curation/explore.mjs Normal file
View File

@@ -0,0 +1,652 @@
#!/usr/bin/env node
/**
* Phase 1: Database Exploration — Single-Query Version
*
* Runs ALL analytics in ONE SQL query to handle unstable connections.
*
* Usage:
* PGSSLMODE=disable DATABASE_URL=postgres://... node scripts/curation/explore.mjs
*/
import { createRequire } from 'module';
import { writeFileSync } from 'fs';
import { resolve, dirname } from 'path';
import { fileURLToPath } from 'url';
const __dirname = dirname(fileURLToPath(import.meta.url));
const projectRoot = resolve(__dirname, '../..');
let pg;
const tryPaths = [
'/tmp/package.json',
process.env.APPDATA ? resolve(process.env.APPDATA, '..', 'Local', 'Temp', 'package.json') : null,
resolve(projectRoot, 'package.json'),
].filter(Boolean);
for (const p of tryPaths) {
try { pg = createRequire(p)('pg'); break; } catch {}
}
if (!pg) { console.error('pg not found. Run: cd /tmp && npm install pg'); process.exit(1); }
const DATABASE_URL = process.env.DATABASE_URL || 'postgresql://postgres:postgres@localhost:5432/skillhub';
process.on('uncaughtException', e => {
if (e.code === 'ECONNRESET') return;
console.error('Fatal:', e); process.exit(1);
});
// ─── Mega Query ──────────────────────────────────────────
// All analytics combined into one JSON result
const MEGA_SQL = `
WITH active AS (
SELECT * FROM skills WHERE is_blocked = false
),
-- 1. Overall
overall AS (
SELECT json_build_object(
'total', (SELECT COUNT(*)::int FROM skills),
'active', (SELECT COUNT(*)::int FROM active),
'blocked', (SELECT COUNT(*) FILTER (WHERE is_blocked)::int FROM skills),
'owners', (SELECT COUNT(DISTINCT github_owner)::int FROM active),
'repos', (SELECT COUNT(DISTINCT github_owner||'/'||github_repo)::int FROM active),
'skillmd', (SELECT COUNT(*) FILTER (WHERE source_format='skill.md')::int FROM active),
'other_fmt', (SELECT COUNT(*) FILTER (WHERE source_format!='skill.md')::int FROM active),
'downloads', (SELECT COALESCE(SUM(download_count),0)::bigint FROM active),
'views', (SELECT COALESCE(SUM(view_count),0)::bigint FROM active),
'rated', (SELECT COUNT(*) FILTER (WHERE rating_count>0)::int FROM active),
'first', (SELECT MIN(created_at)::text FROM active),
'last', (SELECT MAX(created_at)::text FROM active)
) AS data
),
-- 1b. Source formats
source_formats AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT COALESCE(source_format,'null') AS format, COUNT(*)::int AS count,
COUNT(*) FILTER (WHERE is_blocked=false)::int AS active
FROM skills GROUP BY source_format ORDER BY count DESC
) t
),
-- 2. Stars distribution
stars_dist AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT CASE
WHEN github_stars IS NULL OR github_stars=0 THEN '0'
WHEN github_stars BETWEEN 1 AND 5 THEN '1-5'
WHEN github_stars BETWEEN 6 AND 10 THEN '6-10'
WHEN github_stars BETWEEN 11 AND 50 THEN '11-50'
WHEN github_stars BETWEEN 51 AND 100 THEN '51-100'
WHEN github_stars BETWEEN 101 AND 500 THEN '101-500'
WHEN github_stars BETWEEN 501 AND 1000 THEN '501-1K'
WHEN github_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
WHEN github_stars BETWEEN 5001 AND 10000 THEN '5K-10K'
WHEN github_stars BETWEEN 10001 AND 50000 THEN '10K-50K'
ELSE '50K+'
END AS stars, COUNT(*)::int AS skills,
COUNT(DISTINCT github_owner)::int AS owners,
COUNT(DISTINCT github_owner||'/'||github_repo)::int AS repos
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(github_stars,0))
) t
),
-- 3. Downloads
dl_dist AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT CASE
WHEN COALESCE(download_count,0)=0 THEN '0'
WHEN download_count BETWEEN 1 AND 5 THEN '1-5'
WHEN download_count BETWEEN 6 AND 50 THEN '6-50'
WHEN download_count BETWEEN 51 AND 500 THEN '51-500'
ELSE '500+'
END AS range, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(download_count,0))
) t
),
-- 3b. Views
vw_dist AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT CASE
WHEN COALESCE(view_count,0)=0 THEN '0'
WHEN view_count BETWEEN 1 AND 10 THEN '1-10'
WHEN view_count BETWEEN 11 AND 100 THEN '11-100'
WHEN view_count BETWEEN 101 AND 1000 THEN '101-1K'
ELSE '1K+'
END AS range, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(view_count,0))
) t
),
-- 3c. Top downloaded
top_dl AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT id, name, download_count AS dl, view_count AS views,
github_stars AS stars, github_owner AS owner
FROM active WHERE COALESCE(download_count,0)>0
ORDER BY download_count DESC LIMIT 20
) t
),
-- 3d. Top viewed
top_vw AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT id, name, view_count AS views, download_count AS dl,
github_stars AS stars, github_owner AS owner
FROM active WHERE COALESCE(view_count,0)>0
ORDER BY view_count DESC LIMIT 20
) t
),
-- 4. Security
security AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT COALESCE(security_status,'null') AS status, COUNT(*)::int AS count
FROM active GROUP BY security_status ORDER BY count DESC
) t
),
-- 5. Content stats
content_stats AS (
SELECT json_build_object(
'has_content', COUNT(*) FILTER (WHERE raw_content IS NOT NULL)::int,
'no_content', COUNT(*) FILTER (WHERE raw_content IS NULL)::int,
'avg_len', ROUND(AVG(LENGTH(raw_content)) FILTER (WHERE raw_content IS NOT NULL))::int,
'max_len', MAX(LENGTH(raw_content))::int,
'median_len', PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY LENGTH(raw_content))
FILTER (WHERE raw_content IS NOT NULL)::int,
'avg_desc', ROUND(AVG(LENGTH(description)))::int,
'good_desc', COUNT(*) FILTER (WHERE LENGTH(description)>100)::int,
'short_desc', COUNT(*) FILTER (WHERE LENGTH(description)<=20)::int,
'has_ver', COUNT(*) FILTER (WHERE version IS NOT NULL)::int,
'has_lic', COUNT(*) FILTER (WHERE license IS NOT NULL)::int,
'has_auth', COUNT(*) FILTER (WHERE author IS NOT NULL)::int,
'has_hp', COUNT(*) FILTER (WHERE homepage IS NOT NULL)::int
) AS data FROM active
),
-- 5b. Content length dist
content_len AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT CASE
WHEN raw_content IS NULL THEN 'null'
WHEN LENGTH(raw_content)=0 THEN 'empty'
WHEN LENGTH(raw_content)<200 THEN '<200'
WHEN LENGTH(raw_content)<500 THEN '200-500'
WHEN LENGTH(raw_content)<1000 THEN '500-1K'
WHEN LENGTH(raw_content)<3000 THEN '1K-3K'
WHEN LENGTH(raw_content)<5000 THEN '3K-5K'
WHEN LENGTH(raw_content)<10000 THEN '5K-10K'
WHEN LENGTH(raw_content)<50000 THEN '10K-50K'
ELSE '50K+'
END AS range, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(LENGTH(raw_content),-1))
) t
),
-- 6. Triggers
trigger_stats AS (
SELECT json_build_object(
'has_triggers', COUNT(*) FILTER (WHERE triggers IS NOT NULL)::int,
'no_triggers', COUNT(*) FILTER (WHERE triggers IS NULL)::int,
'file_pats', COUNT(*) FILTER (WHERE triggers->>'filePatterns' IS NOT NULL
AND triggers->>'filePatterns'!='[]' AND triggers->>'filePatterns'!='null')::int,
'keywords', COUNT(*) FILTER (WHERE triggers->>'keywords' IS NOT NULL
AND triggers->>'keywords'!='[]' AND triggers->>'keywords'!='null')::int,
'languages', COUNT(*) FILTER (WHERE triggers->>'languages' IS NOT NULL
AND triggers->>'languages'!='[]' AND triggers->>'languages'!='null')::int,
'has_compat', COUNT(*) FILTER (WHERE compatibility IS NOT NULL)::int,
'platforms', COUNT(*) FILTER (WHERE compatibility->>'platforms' IS NOT NULL
AND compatibility->>'platforms'!='[]' AND compatibility->>'platforms'!='null')::int
) AS data FROM active
),
-- 7. Freshness
freshness AS (
SELECT json_build_object(
'created', (SELECT json_agg(row_to_json(t)) FROM (
SELECT CASE
WHEN created_at>NOW()-INTERVAL '7 days' THEN 'Last 7d'
WHEN created_at>NOW()-INTERVAL '30 days' THEN 'Last 30d'
WHEN created_at>NOW()-INTERVAL '90 days' THEN 'Last 90d'
WHEN created_at>NOW()-INTERVAL '180 days' THEN 'Last 180d'
ELSE 'Older'
END AS range, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY MIN(NOW()-created_at)
) t),
'updated', (SELECT json_agg(row_to_json(t)) FROM (
SELECT CASE
WHEN updated_at>NOW()-INTERVAL '7 days' THEN 'Last 7d'
WHEN updated_at>NOW()-INTERVAL '30 days' THEN 'Last 30d'
WHEN updated_at>NOW()-INTERVAL '90 days' THEN 'Last 90d'
WHEN updated_at>NOW()-INTERVAL '180 days' THEN 'Last 180d'
ELSE 'Older'
END AS range, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY MIN(NOW()-updated_at)
) t),
'last_dl', (SELECT json_agg(row_to_json(t)) FROM (
SELECT CASE
WHEN last_downloaded_at IS NULL THEN 'Never'
WHEN last_downloaded_at>NOW()-INTERVAL '30 days' THEN 'Last 30d'
WHEN last_downloaded_at>NOW()-INTERVAL '90 days' THEN 'Last 90d'
ELSE 'Older'
END AS range, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(last_downloaded_at,'1970-01-01'::timestamp))
) t)
) AS data
),
-- 8. Top owners
top_owners_count AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT github_owner AS owner, COUNT(*)::int AS skills,
COUNT(DISTINCT github_repo)::int AS repos,
MAX(github_stars)::int AS max_stars,
COALESCE(SUM(download_count),0)::int AS dl
FROM active GROUP BY github_owner ORDER BY skills DESC LIMIT 30
) t
),
top_owners_stars AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT github_owner AS owner, MAX(github_stars)::int AS max_stars,
COUNT(*)::int AS skills, COUNT(DISTINCT github_repo)::int AS repos
FROM active GROUP BY github_owner ORDER BY max_stars DESC LIMIT 20
) t
),
owner_concentration AS (
SELECT json_build_object(
'top1', SUM(cnt) FILTER (WHERE rn<=1)::int,
'top5', SUM(cnt) FILTER (WHERE rn<=5)::int,
'top10', SUM(cnt) FILTER (WHERE rn<=10)::int,
'top20', SUM(cnt) FILTER (WHERE rn<=20)::int,
'top50', SUM(cnt) FILTER (WHERE rn<=50)::int,
'total', SUM(cnt)::int,
'owners', COUNT(*)::int
) AS data FROM (
SELECT github_owner, COUNT(*)::int AS cnt,
ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn
FROM active GROUP BY github_owner
) x
),
-- 9. Multi-skill repos
aggregators AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT github_owner||'/'||github_repo AS repo, COUNT(*)::int AS skills,
MAX(github_stars)::int AS stars
FROM active GROUP BY github_owner, github_repo HAVING COUNT(*)>=20
ORDER BY skills DESC LIMIT 30
) t
),
collections AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT github_owner||'/'||github_repo AS repo, COUNT(*)::int AS skills,
MAX(github_stars)::int AS stars
FROM active GROUP BY github_owner, github_repo HAVING COUNT(*) BETWEEN 3 AND 19
ORDER BY skills DESC LIMIT 30
) t
),
skills_per_repo AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
WITH rc AS (SELECT COUNT(*)::int AS cnt FROM active GROUP BY github_owner, github_repo)
SELECT CASE
WHEN cnt=1 THEN '1' WHEN cnt=2 THEN '2'
WHEN cnt BETWEEN 3 AND 5 THEN '3-5' WHEN cnt BETWEEN 6 AND 10 THEN '6-10'
WHEN cnt BETWEEN 11 AND 20 THEN '11-20' WHEN cnt BETWEEN 21 AND 50 THEN '21-50'
WHEN cnt BETWEEN 51 AND 100 THEN '51-100' ELSE '100+'
END AS per_repo, COUNT(*)::int AS repos, SUM(cnt)::int AS total_skills
FROM rc GROUP BY 1 ORDER BY MIN(cnt)
) t
),
-- 10. Categories
categories_dist AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT c.name AS category, c.skill_count::int AS skills
FROM categories c WHERE c.id NOT LIKE 'parent-%' ORDER BY c.skill_count DESC
) t
),
uncategorized AS (
SELECT COUNT(*)::int AS data FROM active s
WHERE NOT EXISTS (SELECT 1 FROM skill_categories sc WHERE sc.skill_id=s.id)
),
-- 11. Flags
flags AS (
SELECT json_build_object(
'verified', COUNT(*) FILTER (WHERE is_verified)::int,
'featured', COUNT(*) FILTER (WHERE is_featured)::int,
'user_rated', COUNT(*) FILTER (WHERE rating_count>0)::int,
'avg_rating', ROUND(AVG(rating) FILTER (WHERE rating IS NOT NULL),2)::numeric,
'max_ratings', MAX(rating_count)::int
) AS data FROM active
),
-- 12. Cross-table
cross_stats AS (
SELECT json_build_object(
'users', (SELECT COUNT(*)::int FROM users),
'ratings', (SELECT COUNT(*)::int FROM ratings),
'installs', (SELECT COUNT(*)::int FROM installations),
'favorites', (SELECT COUNT(*)::int FROM favorites),
'disc_repos', (SELECT COUNT(*)::int FROM discovered_repos),
'disc_with_skills', (SELECT COUNT(*) FILTER (WHERE has_skill_md)::int FROM discovered_repos),
'removals', (SELECT COUNT(*)::int FROM removal_requests),
'adds', (SELECT COUNT(*)::int FROM add_requests),
'subs', (SELECT COUNT(*) FILTER (WHERE unsubscribed_at IS NULL)::int FROM email_subscriptions)
) AS data
),
install_platform AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT platform, COUNT(*)::int AS count FROM installations GROUP BY platform ORDER BY count DESC
) t
),
-- 13. Usability signals
usability AS (
SELECT json_build_object(
'strong', COUNT(*) FILTER (WHERE LENGTH(description)>50 AND raw_content IS NOT NULL
AND LENGTH(raw_content)>500 AND security_status='pass')::int,
'file_triggers', COUNT(*) FILTER (WHERE triggers IS NOT NULL AND triggers!='{}'
AND triggers->>'filePatterns' IS NOT NULL AND triggers->>'filePatterns'!='[]')::int,
'downloaded', COUNT(*) FILTER (WHERE COALESCE(download_count,0)>0)::int,
'high_q', COUNT(*) FILTER (WHERE github_stars>=10 AND LENGTH(description)>50
AND raw_content IS NOT NULL AND security_status='pass')::int,
'premium', COUNT(*) FILTER (WHERE github_stars>=100 AND COALESCE(download_count,0)>0
AND security_status='pass')::int,
'skillmd_q', COUNT(*) FILTER (WHERE source_format='skill.md' AND LENGTH(description)>50
AND raw_content IS NOT NULL AND LENGTH(raw_content)>300 AND security_status='pass')::int
) AS data FROM active
),
-- 14. Standalone vs project-bound
repo_types AS (
SELECT json_build_object(
'single', COUNT(*) FILTER (WHERE cnt=1)::int,
'two', COUNT(*) FILTER (WHERE cnt=2)::int,
'multi', COUNT(*) FILTER (WHERE cnt>=3)::int,
'total', COUNT(*)::int
) AS data FROM (
SELECT COUNT(*)::int AS cnt FROM active GROUP BY github_owner, github_repo
) x
),
name_patterns AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT CASE
WHEN name ILIKE '%rule%' OR name ILIKE '%config%' OR name ILIKE '%setup%' THEN 'rules/config/setup'
WHEN name ILIKE '%my-%' OR name ILIKE '%my_%' THEN 'my-prefix'
WHEN name ILIKE '%cursor%' OR name ILIKE '%claude%' OR name ILIKE '%copilot%' THEN 'tool-specific'
WHEN name ILIKE '%project%' OR name ILIKE '%team%' OR name ILIKE '%internal%' THEN 'project/team'
WHEN name ILIKE '%.mdc' OR name ILIKE '%cursorrule%' THEN 'cursor-rules'
ELSE 'generic'
END AS pattern, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY skills DESC
) t
),
-- 15. Samples
top_by_stars AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT id, name, github_stars AS stars, COALESCE(download_count,0) AS dl,
security_status AS sec, source_format AS fmt, LEFT(description,80) AS description
FROM active ORDER BY github_stars DESC LIMIT 15
) t
),
top_by_dl AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT id, name, download_count AS dl, view_count AS views,
github_stars AS stars, LEFT(description,80) AS description
FROM active WHERE COALESCE(download_count,0)>0
ORDER BY download_count DESC LIMIT 15
) t
),
-- 16. Discovered repos
discovered AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT discovered_via AS source, COUNT(*)::int AS total,
COUNT(*) FILTER (WHERE has_skill_md)::int AS with_skills,
COUNT(*) FILTER (WHERE last_scanned IS NOT NULL)::int AS scanned
FROM discovered_repos GROUP BY discovered_via ORDER BY total DESC
) t
),
-- 18. Cache
cache_stats AS (
SELECT json_build_object(
'cached', COUNT(*) FILTER (WHERE cached_files IS NOT NULL)::int,
'not_cached', COUNT(*) FILTER (WHERE cached_files IS NULL)::int
) AS data FROM active
),
-- 19. Branches
branches AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT COALESCE(branch,'null') AS branch, COUNT(*)::int AS count
FROM active GROUP BY branch ORDER BY count DESC LIMIT 10
) t
),
-- 20. Duplicates
exact_dupes AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT name, LEFT(description,60) AS desc, COUNT(*)::int AS copies,
STRING_AGG(DISTINCT github_owner,', ') AS owners
FROM active WHERE description IS NOT NULL AND LENGTH(description)>10
GROUP BY name, description HAVING COUNT(*)>1
ORDER BY copies DESC LIMIT 15
) t
),
hash_dupes AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT content_hash, COUNT(*)::int AS copies,
MIN(name) AS sample, STRING_AGG(DISTINCT github_owner,', ') AS owners
FROM active WHERE content_hash IS NOT NULL
GROUP BY content_hash HAVING COUNT(*)>2
ORDER BY copies DESC LIMIT 15
) t
)
-- Final: combine everything into one JSON
SELECT json_build_object(
'overall', (SELECT data FROM overall),
'sourceFormats', (SELECT data FROM source_formats),
'starsDist', (SELECT data FROM stars_dist),
'dlDist', (SELECT data FROM dl_dist),
'vwDist', (SELECT data FROM vw_dist),
'topDL', (SELECT data FROM top_dl),
'topVW', (SELECT data FROM top_vw),
'security', (SELECT data FROM security),
'contentStats', (SELECT data FROM content_stats),
'contentLen', (SELECT data FROM content_len),
'triggerStats', (SELECT data FROM trigger_stats),
'freshness', (SELECT data FROM freshness),
'topOwnersCount', (SELECT data FROM top_owners_count),
'topOwnersStars', (SELECT data FROM top_owners_stars),
'ownerConcentration', (SELECT data FROM owner_concentration),
'aggregators', (SELECT data FROM aggregators),
'collections', (SELECT data FROM collections),
'skillsPerRepo', (SELECT data FROM skills_per_repo),
'categories', (SELECT data FROM categories_dist),
'uncategorized', (SELECT data FROM uncategorized),
'flags', (SELECT data FROM flags),
'crossStats', (SELECT data FROM cross_stats),
'installPlatform', (SELECT data FROM install_platform),
'usability', (SELECT data FROM usability),
'repoTypes', (SELECT data FROM repo_types),
'namePatterns', (SELECT data FROM name_patterns),
'topByStars', (SELECT data FROM top_by_stars),
'topByDL', (SELECT data FROM top_by_dl),
'discovered', (SELECT data FROM discovered),
'cacheStats', (SELECT data FROM cache_stats),
'branches', (SELECT data FROM branches),
'exactDupes', (SELECT data FROM exact_dupes),
'hashDupes', (SELECT data FROM hash_dupes)
) AS report;
`;
// ─── Helpers ─────────────────────────────────────────────
function header(title) {
console.log(`\n${'='.repeat(70)}\n ${title}\n${'='.repeat(70)}`);
}
function sub(title) {
console.log(`\n -- ${title} ${'─'.repeat(Math.max(0, 58 - title.length))}`);
}
function tbl(rows, max = 30) {
if (!rows || rows.length === 0) { console.log(' (no data)'); return; }
const display = rows.slice(0, max);
const keys = Object.keys(display[0]);
const w = keys.map(k => Math.max(k.length, ...display.map(r => String(r[k] ?? '').length)));
console.log(' ' + keys.map((k, i) => k.padEnd(w[i])).join(' '));
console.log(' ' + w.map(x => '-'.repeat(x)).join(' '));
for (const row of display)
console.log(' ' + keys.map((k, i) => String(row[k] ?? '').padEnd(w[i])).join(' '));
if (rows.length > max) console.log(` ... and ${rows.length - max} more`);
}
const n = v => Number(v ?? 0).toLocaleString('en-US');
const p = (a, b) => (Number(b ?? 1) === 0 ? '0.0%' : (Number(a ?? 0) / Number(b) * 100).toFixed(1) + '%');
// ─── Run ─────────────────────────────────────────────────
async function run() {
const t0 = Date.now();
console.log('Connecting and running mega-query (this may take 30-60s)...');
const c = new pg.Client({ connectionString: DATABASE_URL, ssl: false,
connectionTimeoutMillis: 15000, query_timeout: 120000,
keepAlive: true, keepAliveInitialDelayMillis: 1000 });
await c.connect();
const result = await c.query(MEGA_SQL);
await c.end().catch(() => {});
const r = result.rows[0].report;
const dur = ((Date.now() - t0) / 1000).toFixed(1);
console.log(`Query completed in ${dur}s\n`);
// ── Display ──
const total = r.overall.active;
header('1. OVERALL STATISTICS');
const ov = r.overall;
console.log(` Total skills: ${n(ov.total)}`);
console.log(` Active (not blocked): ${n(ov.active)}`);
console.log(` Blocked: ${n(ov.blocked)}`);
console.log(` Unique owners: ${n(ov.owners)}`);
console.log(` Unique repos: ${n(ov.repos)}`);
console.log(` SKILL.md format: ${n(ov.skillmd)} (${p(ov.skillmd, ov.total)})`);
console.log(` Other formats: ${n(ov.other_fmt)} (${p(ov.other_fmt, ov.total)})`);
console.log(` Total downloads: ${n(ov.downloads)}`);
console.log(` Total views: ${n(ov.views)}`);
console.log(` Rated skills: ${n(ov.rated)}`);
console.log(` Date range: ${ov.first} -> ${ov.last}`);
sub('Source Formats');
tbl(r.sourceFormats);
header('2. GITHUB STARS');
tbl(r.starsDist);
header('3. ENGAGEMENT');
sub('Downloads'); tbl(r.dlDist);
sub('Views'); tbl(r.vwDist);
sub('Top 20 Downloaded'); tbl(r.topDL);
sub('Top 20 Viewed'); tbl(r.topVW);
header('4. SECURITY');
tbl(r.security);
header('5. CONTENT');
const cs = r.contentStats;
console.log(` Has content: ${n(cs.has_content)} (${p(cs.has_content, total)})`);
console.log(` No content: ${n(cs.no_content)} (${p(cs.no_content, total)})`);
console.log(` Avg length: ${n(cs.avg_len)} | Median: ${n(cs.median_len)} | Max: ${n(cs.max_len)}`);
console.log(` Avg desc: ${n(cs.avg_desc)} | Good(>100): ${n(cs.good_desc)} | Short(<=20): ${n(cs.short_desc)}`);
console.log(` Version: ${n(cs.has_ver)} | License: ${n(cs.has_lic)} | Author: ${n(cs.has_auth)} | HP: ${n(cs.has_hp)}`);
sub('Content Length Dist');
tbl(r.contentLen);
header('6. TRIGGERS');
const tr = r.triggerStats;
console.log(` Has triggers: ${n(tr.has_triggers)} (${p(tr.has_triggers, total)})`);
console.log(` filePatterns: ${n(tr.file_pats)} | keywords: ${n(tr.keywords)} | languages: ${n(tr.languages)}`);
console.log(` Compatibility: ${n(tr.has_compat)} | platforms: ${n(tr.platforms)}`);
header('7. FRESHNESS');
sub('Created'); tbl(r.freshness.created);
sub('Updated'); tbl(r.freshness.updated);
sub('Last DL'); tbl(r.freshness.last_dl);
header('8. TOP OWNERS');
sub('By Skill Count'); tbl(r.topOwnersCount);
sub('By Stars'); tbl(r.topOwnersStars);
sub('Concentration');
const oc = r.ownerConcentration;
console.log(` Top 1: ${n(oc.top1)} (${p(oc.top1, oc.total)}) | Top 5: ${n(oc.top5)} (${p(oc.top5, oc.total)}) | Top 10: ${n(oc.top10)} (${p(oc.top10, oc.total)})`);
console.log(` Top 20: ${n(oc.top20)} (${p(oc.top20, oc.total)}) | Top 50: ${n(oc.top50)} (${p(oc.top50, oc.total)}) | Total owners: ${n(oc.owners)}`);
header('9. MULTI-SKILL REPOS');
sub('Aggregators (20+)'); tbl(r.aggregators);
sub('Collections (3-19)'); tbl(r.collections);
sub('Skills-per-Repo'); tbl(r.skillsPerRepo);
header('10. CATEGORIES');
tbl(r.categories);
console.log(`\n Uncategorized: ${n(r.uncategorized)}`);
header('11. FLAGS');
const fl = r.flags;
console.log(` Verified: ${n(fl.verified)} | Featured: ${n(fl.featured)} | Rated: ${n(fl.user_rated)} | Avg: ${fl.avg_rating ?? 'N/A'}`);
header('12. CROSS-TABLE');
const xt = r.crossStats;
console.log(` Users: ${n(xt.users)} | Ratings: ${n(xt.ratings)} | Installs: ${n(xt.installs)} | Favs: ${n(xt.favorites)}`);
console.log(` Discovered: ${n(xt.disc_repos)} (${n(xt.disc_with_skills)} with skills) | Subs: ${n(xt.subs)}`);
sub('Install Platform'); tbl(r.installPlatform);
header('13. USABILITY SIGNALS');
const us = r.usability;
console.log(` Strong standalone: ${n(us.strong)} (desc>50 + content>500 + sec=pass)`);
console.log(` File triggers: ${n(us.file_triggers)}`);
console.log(` Ever downloaded: ${n(us.downloaded)}`);
console.log(` High quality: ${n(us.high_q)} (stars>=10 + desc + sec)`);
console.log(` Premium: ${n(us.premium)} (stars>=100 + dl>0 + sec)`);
console.log(` SKILL.md quality: ${n(us.skillmd_q)}`);
header('14. STANDALONE vs PROJECT-BOUND');
const sm = r.repoTypes;
console.log(` Single: ${n(sm.single)} (${p(sm.single,sm.total)}) | Two: ${n(sm.two)} | Multi: ${n(sm.multi)} | Total repos: ${n(sm.total)}`);
sub('Name Patterns'); tbl(r.namePatterns);
header('15. SAMPLES');
sub('Top by Stars'); tbl(r.topByStars);
sub('Top by DL'); tbl(r.topByDL);
header('16. DISCOVERED REPOS');
tbl(r.discovered);
header('17. CACHE');
const cf = r.cacheStats;
console.log(` Cached: ${n(cf.cached)} (${p(cf.cached,total)}) | Not: ${n(cf.not_cached)}`);
header('19. BRANCHES');
tbl(r.branches);
header('20. DUPLICATES');
sub('Same Name+Desc'); tbl(r.exactDupes);
sub('Same Hash (3+)'); tbl(r.hashDupes);
header(`COMPLETE (${dur}s)`);
// Save JSON
r.generatedAt = new Date().toISOString();
r.durationSeconds = parseFloat(dur);
const reportPath = resolve(__dirname, 'explore-report.json');
writeFileSync(reportPath, JSON.stringify(r, null, 2), 'utf-8');
console.log(`\n Report saved: ${reportPath}`);
}
run().catch(async e => {
console.error('FAILED:', e.message || e);
process.exit(1);
});

258
scripts/curation/explore.sql Normal file
View File

@@ -0,0 +1,258 @@
-- Phase 1: Database Exploration - Run inside container:
-- psql -U postgres skillhub -t -A -f /tmp/explore.sql > /tmp/report.json
-- Or paste this SQL directly in psql
WITH active AS (SELECT * FROM skills WHERE is_blocked = false),
overall AS (
SELECT json_build_object(
'total', (SELECT COUNT(*)::int FROM skills),
'active', (SELECT COUNT(*)::int FROM active),
'blocked', (SELECT COUNT(*) FILTER (WHERE is_blocked)::int FROM skills),
'owners', (SELECT COUNT(DISTINCT github_owner)::int FROM active),
'repos', (SELECT COUNT(DISTINCT github_owner||'/'||github_repo)::int FROM active),
'skillmd', (SELECT COUNT(*) FILTER (WHERE source_format='skill.md')::int FROM active),
'other_fmt', (SELECT COUNT(*) FILTER (WHERE source_format!='skill.md')::int FROM active),
'downloads', (SELECT COALESCE(SUM(download_count),0)::bigint FROM active),
'views', (SELECT COALESCE(SUM(view_count),0)::bigint FROM active),
'rated', (SELECT COUNT(*) FILTER (WHERE rating_count>0)::int FROM active),
'first', (SELECT MIN(created_at)::text FROM active),
'last', (SELECT MAX(created_at)::text FROM active)
) AS data
),
source_formats AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT COALESCE(source_format,'null') AS format, COUNT(*)::int AS count
FROM skills GROUP BY source_format ORDER BY count DESC
) t
),
stars_dist AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT CASE
WHEN github_stars IS NULL OR github_stars=0 THEN '0'
WHEN github_stars BETWEEN 1 AND 10 THEN '1-10'
WHEN github_stars BETWEEN 11 AND 50 THEN '11-50'
WHEN github_stars BETWEEN 51 AND 100 THEN '51-100'
WHEN github_stars BETWEEN 101 AND 500 THEN '101-500'
WHEN github_stars BETWEEN 501 AND 1000 THEN '501-1K'
WHEN github_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
WHEN github_stars BETWEEN 5001 AND 50000 THEN '5K-50K'
ELSE '50K+'
END AS stars, COUNT(*)::int AS skills,
COUNT(DISTINCT github_owner)::int AS owners
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(github_stars,0))
) t
),
dl_dist AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT CASE
WHEN COALESCE(download_count,0)=0 THEN '0'
WHEN download_count BETWEEN 1 AND 10 THEN '1-10'
WHEN download_count BETWEEN 11 AND 100 THEN '11-100'
ELSE '100+'
END AS range, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(download_count,0))
) t
),
top_dl AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT id, name, download_count AS dl, github_stars AS stars, github_owner AS owner
FROM active WHERE COALESCE(download_count,0)>0 ORDER BY download_count DESC LIMIT 20
) t
),
top_vw AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT id, name, view_count AS views, github_stars AS stars, github_owner AS owner
FROM active WHERE COALESCE(view_count,0)>0 ORDER BY view_count DESC LIMIT 20
) t
),
security AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT COALESCE(security_status,'null') AS status, COUNT(*)::int AS count
FROM active GROUP BY security_status ORDER BY count DESC
) t
),
content_stats AS (
SELECT json_build_object(
'has_content', COUNT(*) FILTER (WHERE raw_content IS NOT NULL)::int,
'no_content', COUNT(*) FILTER (WHERE raw_content IS NULL)::int,
'avg_len', ROUND(AVG(LENGTH(raw_content)) FILTER (WHERE raw_content IS NOT NULL))::int,
'max_len', MAX(LENGTH(raw_content))::int,
'median_len', PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY LENGTH(raw_content)) FILTER (WHERE raw_content IS NOT NULL)::int,
'avg_desc', ROUND(AVG(LENGTH(description)))::int,
'good_desc', COUNT(*) FILTER (WHERE LENGTH(description)>100)::int,
'short_desc', COUNT(*) FILTER (WHERE LENGTH(description)<=20)::int,
'has_ver', COUNT(*) FILTER (WHERE version IS NOT NULL)::int,
'has_lic', COUNT(*) FILTER (WHERE license IS NOT NULL)::int,
'has_auth', COUNT(*) FILTER (WHERE author IS NOT NULL)::int
) AS data FROM active
),
content_len AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT CASE
WHEN raw_content IS NULL THEN 'null'
WHEN LENGTH(raw_content)<200 THEN '<200'
WHEN LENGTH(raw_content)<1000 THEN '200-1K'
WHEN LENGTH(raw_content)<5000 THEN '1K-5K'
WHEN LENGTH(raw_content)<10000 THEN '5K-10K'
WHEN LENGTH(raw_content)<50000 THEN '10K-50K'
ELSE '50K+'
END AS range, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(LENGTH(raw_content),-1))
) t
),
trigger_stats AS (
SELECT json_build_object(
'has_triggers', COUNT(*) FILTER (WHERE triggers IS NOT NULL)::int,
'no_triggers', COUNT(*) FILTER (WHERE triggers IS NULL)::int,
'file_pats', COUNT(*) FILTER (WHERE triggers->>'filePatterns' IS NOT NULL AND triggers->>'filePatterns'!='[]')::int,
'keywords', COUNT(*) FILTER (WHERE triggers->>'keywords' IS NOT NULL AND triggers->>'keywords'!='[]')::int,
'platforms', COUNT(*) FILTER (WHERE compatibility->>'platforms' IS NOT NULL AND compatibility->>'platforms'!='[]')::int
) AS data FROM active
),
freshness AS (
SELECT json_build_object(
'created', (SELECT json_agg(row_to_json(t)) FROM (
SELECT CASE WHEN created_at>NOW()-INTERVAL '7d' THEN '7d' WHEN created_at>NOW()-INTERVAL '30d' THEN '30d'
WHEN created_at>NOW()-INTERVAL '90d' THEN '90d' ELSE 'older' END AS range, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY MIN(NOW()-created_at)) t),
'updated', (SELECT json_agg(row_to_json(t)) FROM (
SELECT CASE WHEN updated_at>NOW()-INTERVAL '7d' THEN '7d' WHEN updated_at>NOW()-INTERVAL '30d' THEN '30d'
WHEN updated_at>NOW()-INTERVAL '90d' THEN '90d' ELSE 'older' END AS range, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY MIN(NOW()-updated_at)) t)
) AS data
),
top_owners AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT github_owner AS owner, COUNT(*)::int AS skills, COUNT(DISTINCT github_repo)::int AS repos,
MAX(github_stars)::int AS max_stars, COALESCE(SUM(download_count),0)::int AS dl
FROM active GROUP BY github_owner ORDER BY skills DESC LIMIT 30
) t
),
owner_concentration AS (
SELECT json_build_object(
'top1', SUM(cnt) FILTER (WHERE rn<=1)::int, 'top5', SUM(cnt) FILTER (WHERE rn<=5)::int,
'top10', SUM(cnt) FILTER (WHERE rn<=10)::int, 'top20', SUM(cnt) FILTER (WHERE rn<=20)::int,
'total', SUM(cnt)::int, 'owners', COUNT(*)::int
) AS data FROM (
SELECT github_owner, COUNT(*)::int AS cnt, ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn
FROM active GROUP BY github_owner
) x
),
aggregators AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT github_owner||'/'||github_repo AS repo, COUNT(*)::int AS skills, MAX(github_stars)::int AS stars
FROM active GROUP BY github_owner, github_repo HAVING COUNT(*)>=20 ORDER BY skills DESC LIMIT 20
) t
),
collections AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT github_owner||'/'||github_repo AS repo, COUNT(*)::int AS skills, MAX(github_stars)::int AS stars
FROM active GROUP BY github_owner, github_repo HAVING COUNT(*) BETWEEN 3 AND 19 ORDER BY skills DESC LIMIT 20
) t
),
skills_per_repo AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
WITH rc AS (SELECT COUNT(*)::int AS cnt FROM active GROUP BY github_owner, github_repo)
SELECT CASE WHEN cnt=1 THEN '1' WHEN cnt=2 THEN '2' WHEN cnt BETWEEN 3 AND 10 THEN '3-10'
WHEN cnt BETWEEN 11 AND 50 THEN '11-50' ELSE '50+' END AS per_repo,
COUNT(*)::int AS repos, SUM(cnt)::int AS total_skills
FROM rc GROUP BY 1 ORDER BY MIN(cnt)
) t
),
categories_dist AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT c.name AS category, c.skill_count::int AS skills
FROM categories c WHERE c.id NOT LIKE 'parent-%' ORDER BY c.skill_count DESC
) t
),
flags AS (
SELECT json_build_object(
'verified', COUNT(*) FILTER (WHERE is_verified)::int,
'featured', COUNT(*) FILTER (WHERE is_featured)::int,
'user_rated', COUNT(*) FILTER (WHERE rating_count>0)::int
) AS data FROM active
),
cross_stats AS (
SELECT json_build_object(
'users', (SELECT COUNT(*)::int FROM users),
'ratings', (SELECT COUNT(*)::int FROM ratings),
'installs', (SELECT COUNT(*)::int FROM installations),
'favorites', (SELECT COUNT(*)::int FROM favorites),
'disc_repos', (SELECT COUNT(*)::int FROM discovered_repos),
'subs', (SELECT COUNT(*) FILTER (WHERE unsubscribed_at IS NULL)::int FROM email_subscriptions)
) AS data
),
usability AS (
SELECT json_build_object(
'strong', COUNT(*) FILTER (WHERE LENGTH(description)>50 AND raw_content IS NOT NULL AND LENGTH(raw_content)>500 AND security_status='pass')::int,
'downloaded', COUNT(*) FILTER (WHERE COALESCE(download_count,0)>0)::int,
'high_q', COUNT(*) FILTER (WHERE github_stars>=10 AND LENGTH(description)>50 AND raw_content IS NOT NULL AND security_status='pass')::int,
'premium', COUNT(*) FILTER (WHERE github_stars>=100 AND COALESCE(download_count,0)>0 AND security_status='pass')::int,
'skillmd_q', COUNT(*) FILTER (WHERE source_format='skill.md' AND LENGTH(description)>50 AND raw_content IS NOT NULL AND LENGTH(raw_content)>300 AND security_status='pass')::int
) AS data FROM active
),
repo_types AS (
SELECT json_build_object(
'single', COUNT(*) FILTER (WHERE cnt=1)::int, 'two', COUNT(*) FILTER (WHERE cnt=2)::int,
'multi', COUNT(*) FILTER (WHERE cnt>=3)::int, 'total', COUNT(*)::int
) AS data FROM (SELECT COUNT(*)::int AS cnt FROM active GROUP BY github_owner, github_repo) x
),
name_patterns AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT CASE
WHEN name ILIKE '%rule%' OR name ILIKE '%config%' OR name ILIKE '%setup%' THEN 'rules/config'
WHEN name ILIKE '%cursor%' OR name ILIKE '%claude%' OR name ILIKE '%copilot%' THEN 'tool-specific'
WHEN name ILIKE '%project%' OR name ILIKE '%team%' THEN 'project/team'
ELSE 'generic'
END AS pattern, COUNT(*)::int AS skills
FROM active GROUP BY 1 ORDER BY skills DESC
) t
),
top_by_stars AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT id, name, github_stars AS stars, COALESCE(download_count,0) AS dl,
security_status AS sec, source_format AS fmt, LEFT(description,80) AS description
FROM active ORDER BY github_stars DESC LIMIT 15
) t
),
discovered AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT discovered_via AS source, COUNT(*)::int AS total,
COUNT(*) FILTER (WHERE has_skill_md)::int AS with_skills
FROM discovered_repos GROUP BY discovered_via ORDER BY total DESC
) t
),
hash_dupes AS (
SELECT json_agg(row_to_json(t)) AS data FROM (
SELECT content_hash, COUNT(*)::int AS copies, MIN(name) AS sample
FROM active WHERE content_hash IS NOT NULL
GROUP BY content_hash HAVING COUNT(*)>2 ORDER BY copies DESC LIMIT 15
) t
)
SELECT json_build_object(
'overall', (SELECT data FROM overall),
'sourceFormats', (SELECT data FROM source_formats),
'starsDist', (SELECT data FROM stars_dist),
'dlDist', (SELECT data FROM dl_dist),
'topDL', (SELECT data FROM top_dl),
'topVW', (SELECT data FROM top_vw),
'security', (SELECT data FROM security),
'contentStats', (SELECT data FROM content_stats),
'contentLen', (SELECT data FROM content_len),
'triggerStats', (SELECT data FROM trigger_stats),
'freshness', (SELECT data FROM freshness),
'topOwners', (SELECT data FROM top_owners),
'ownerConcentration', (SELECT data FROM owner_concentration),
'aggregators', (SELECT data FROM aggregators),
'collections', (SELECT data FROM collections),
'skillsPerRepo', (SELECT data FROM skills_per_repo),
'categories', (SELECT data FROM categories_dist),
'flags', (SELECT data FROM flags),
'crossStats', (SELECT data FROM cross_stats),
'usability', (SELECT data FROM usability),
'repoTypes', (SELECT data FROM repo_types),
'namePatterns', (SELECT data FROM name_patterns),
'topByStars', (SELECT data FROM top_by_stars),
'discovered', (SELECT data FROM discovered),
'hashDupes', (SELECT data FROM hash_dupes)
) AS report;

894
scripts/curation/explore.ts Normal file
View File

@@ -0,0 +1,894 @@
#!/usr/bin/env tsx
/**
* Phase 1: Database Exploration for Curation
*
* Comprehensive analysis of all indexed skills to understand:
* - Overall statistics and distribution
* - Quality signals and content analysis
* - Owner/repo concentration
* - Freshness and activity
* - Usability signals for curation decisions
*
* Usage:
* cd services/indexer && npx tsx ../../scripts/curation/explore.ts
*
* Output: prints report to console + saves JSON to scripts/curation/explore-report.json
*/
import { createDb, closeDb, sql } from '@skillhub/db';
import { writeFileSync } from 'fs';
import { resolve, dirname } from 'path';
import { fileURLToPath } from 'url';
// ─── Connection ────────────────────────────────────────────────────────────────
const db = createDb();
// ─── Helpers ───────────────────────────────────────────────────────────────────
function header(title: string) {
const line = '='.repeat(70);
console.log(`\n${line}`);
console.log(` ${title}`);
console.log(line);
}
function subHeader(title: string) {
console.log(`\n -- ${title} ${'─'.repeat(Math.max(0, 60 - title.length))}`);
}
function table(rows: Record<string, unknown>[], maxRows = 30) {
if (rows.length === 0) {
console.log(' (no data)');
return;
}
const display = rows.slice(0, maxRows);
const keys = Object.keys(display[0]);
const widths = keys.map(k =>
Math.max(k.length, ...display.map(r => String(r[k] ?? '').length))
);
// Header
console.log(' ' + keys.map((k, i) => k.padEnd(widths[i])).join(' '));
console.log(' ' + widths.map(w => '-'.repeat(w)).join(' '));
// Rows
for (const row of display) {
console.log(' ' + keys.map((k, i) => String(row[k] ?? '').padEnd(widths[i])).join(' '));
}
if (rows.length > maxRows) {
console.log(` ... and ${rows.length - maxRows} more rows`);
}
}
function num(n: unknown): string {
return Number(n ?? 0).toLocaleString('en-US');
}
function pct(part: unknown, total: unknown): string {
const p = Number(part ?? 0);
const t = Number(total ?? 1);
if (t === 0) return '0.0%';
return (p / t * 100).toFixed(1) + '%';
}
// Helper: execute raw SQL and return rows
async function query<T = Record<string, unknown>>(strings: TemplateStringsArray, ...values: unknown[]): Promise<T[]> {
const result = await db.execute(sql.raw(String.raw(strings, ...values)));
return result.rows as T[];
}
// ─── Report accumulator ────────────────────────────────────────────────────────
const report: Record<string, unknown> = {};
// ─── Queries ───────────────────────────────────────────────────────────────────
async function runExploration() {
const startTime = Date.now();
// ======================================================================
// 1. OVERALL STATISTICS
// ======================================================================
header('1. OVERALL STATISTICS');
const overallRows = await db.execute(sql`
SELECT
COUNT(*)::int AS total_skills,
COUNT(*) FILTER (WHERE is_blocked = false)::int AS active_skills,
COUNT(*) FILTER (WHERE is_blocked = true)::int AS blocked_skills,
COUNT(DISTINCT github_owner)::int AS unique_owners,
COUNT(DISTINCT github_owner || '/' || github_repo)::int AS unique_repos,
COUNT(*) FILTER (WHERE source_format = 'skill.md')::int AS skill_md_count,
COUNT(*) FILTER (WHERE source_format != 'skill.md')::int AS other_format_count,
COALESCE(SUM(download_count), 0)::bigint AS total_downloads,
COALESCE(SUM(view_count), 0)::bigint AS total_views,
COUNT(*) FILTER (WHERE rating_count > 0)::int AS rated_skills,
COALESCE(SUM(rating_count), 0)::int AS total_ratings,
MIN(created_at)::text AS earliest_skill,
MAX(created_at)::text AS latest_skill
FROM skills
`);
const overall = overallRows.rows[0] as Record<string, unknown>;
report.overall = overall;
console.log(` Total skills: ${num(overall.total_skills)}`);
console.log(` Active (not blocked): ${num(overall.active_skills)}`);
console.log(` Blocked: ${num(overall.blocked_skills)}`);
console.log(` Unique owners: ${num(overall.unique_owners)}`);
console.log(` Unique repos: ${num(overall.unique_repos)}`);
console.log(` SKILL.md format: ${num(overall.skill_md_count)} (${pct(overall.skill_md_count, overall.total_skills)})`);
console.log(` Other formats: ${num(overall.other_format_count)} (${pct(overall.other_format_count, overall.total_skills)})`);
console.log(` Total downloads: ${num(overall.total_downloads)}`);
console.log(` Total views: ${num(overall.total_views)}`);
console.log(` Rated skills: ${num(overall.rated_skills)}`);
console.log(` Total ratings: ${num(overall.total_ratings)}`);
console.log(` Date range: ${overall.earliest_skill} -> ${overall.latest_skill}`);
const total = Number(overall.active_skills);
// Source format distribution
subHeader('Source Format Distribution');
const sfRows = await db.execute(sql`
SELECT
COALESCE(source_format, 'null') AS format,
COUNT(*)::int AS count,
COUNT(*) FILTER (WHERE is_blocked = false)::int AS active
FROM skills
GROUP BY source_format
ORDER BY count DESC
`);
report.sourceFormats = sfRows.rows;
table(sfRows.rows as Record<string, unknown>[]);
// ======================================================================
// 2. STARS DISTRIBUTION
// ======================================================================
header('2. GITHUB STARS DISTRIBUTION');
const starsRows = await db.execute(sql`
SELECT
CASE
WHEN github_stars IS NULL OR github_stars = 0 THEN '0 stars'
WHEN github_stars BETWEEN 1 AND 5 THEN '1-5'
WHEN github_stars BETWEEN 6 AND 10 THEN '6-10'
WHEN github_stars BETWEEN 11 AND 50 THEN '11-50'
WHEN github_stars BETWEEN 51 AND 100 THEN '51-100'
WHEN github_stars BETWEEN 101 AND 500 THEN '101-500'
WHEN github_stars BETWEEN 501 AND 1000 THEN '501-1K'
WHEN github_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
WHEN github_stars BETWEEN 5001 AND 10000 THEN '5K-10K'
WHEN github_stars BETWEEN 10001 AND 50000 THEN '10K-50K'
ELSE '50K+'
END AS star_range,
COUNT(*)::int AS skills,
COUNT(DISTINCT github_owner)::int AS owners,
COUNT(DISTINCT github_owner || '/' || github_repo)::int AS repos
FROM skills
WHERE is_blocked = false
GROUP BY
CASE
WHEN github_stars IS NULL OR github_stars = 0 THEN 0
WHEN github_stars BETWEEN 1 AND 5 THEN 1
WHEN github_stars BETWEEN 6 AND 10 THEN 2
WHEN github_stars BETWEEN 11 AND 50 THEN 3
WHEN github_stars BETWEEN 51 AND 100 THEN 4
WHEN github_stars BETWEEN 101 AND 500 THEN 5
WHEN github_stars BETWEEN 501 AND 1000 THEN 6
WHEN github_stars BETWEEN 1001 AND 5000 THEN 7
WHEN github_stars BETWEEN 5001 AND 10000 THEN 8
WHEN github_stars BETWEEN 10001 AND 50000 THEN 9
ELSE 10
END,
CASE
WHEN github_stars IS NULL OR github_stars = 0 THEN '0 stars'
WHEN github_stars BETWEEN 1 AND 5 THEN '1-5'
WHEN github_stars BETWEEN 6 AND 10 THEN '6-10'
WHEN github_stars BETWEEN 11 AND 50 THEN '11-50'
WHEN github_stars BETWEEN 51 AND 100 THEN '51-100'
WHEN github_stars BETWEEN 101 AND 500 THEN '101-500'
WHEN github_stars BETWEEN 501 AND 1000 THEN '501-1K'
WHEN github_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
WHEN github_stars BETWEEN 5001 AND 10000 THEN '5K-10K'
WHEN github_stars BETWEEN 10001 AND 50000 THEN '10K-50K'
ELSE '50K+'
END
ORDER BY MIN(COALESCE(github_stars, 0))
`);
report.starsDist = starsRows.rows;
table(starsRows.rows as Record<string, unknown>[]);
// ======================================================================
// 3. ENGAGEMENT DISTRIBUTION
// ======================================================================
header('3. ENGAGEMENT (Downloads & Views)');
subHeader('Download Distribution');
const dlRows = await db.execute(sql`
SELECT
CASE
WHEN COALESCE(download_count, 0) = 0 THEN '0'
WHEN download_count BETWEEN 1 AND 5 THEN '1-5'
WHEN download_count BETWEEN 6 AND 10 THEN '6-10'
WHEN download_count BETWEEN 11 AND 50 THEN '11-50'
WHEN download_count BETWEEN 51 AND 100 THEN '51-100'
WHEN download_count BETWEEN 101 AND 500 THEN '101-500'
ELSE '500+'
END AS range,
COUNT(*)::int AS skills
FROM skills WHERE is_blocked = false
GROUP BY 1 ORDER BY MIN(COALESCE(download_count, 0))
`);
report.downloadDist = dlRows.rows;
table(dlRows.rows as Record<string, unknown>[]);
subHeader('View Distribution');
const vwRows = await db.execute(sql`
SELECT
CASE
WHEN COALESCE(view_count, 0) = 0 THEN '0'
WHEN view_count BETWEEN 1 AND 10 THEN '1-10'
WHEN view_count BETWEEN 11 AND 50 THEN '11-50'
WHEN view_count BETWEEN 51 AND 100 THEN '51-100'
WHEN view_count BETWEEN 101 AND 500 THEN '101-500'
WHEN view_count BETWEEN 501 AND 1000 THEN '501-1K'
ELSE '1K+'
END AS range,
COUNT(*)::int AS skills
FROM skills WHERE is_blocked = false
GROUP BY 1 ORDER BY MIN(COALESCE(view_count, 0))
`);
report.viewDist = vwRows.rows;
table(vwRows.rows as Record<string, unknown>[]);
subHeader('Top 20 Downloaded Skills');
const topDL = await db.execute(sql`
SELECT id, name, download_count AS downloads, view_count AS views,
github_stars AS stars, github_owner AS owner
FROM skills
WHERE is_blocked = false AND COALESCE(download_count, 0) > 0
ORDER BY download_count DESC LIMIT 20
`);
report.topDownloads = topDL.rows;
table(topDL.rows as Record<string, unknown>[]);
subHeader('Top 20 Viewed Skills');
const topVW = await db.execute(sql`
SELECT id, name, view_count AS views, download_count AS downloads,
github_stars AS stars, github_owner AS owner
FROM skills
WHERE is_blocked = false AND COALESCE(view_count, 0) > 0
ORDER BY view_count DESC LIMIT 20
`);
report.topViews = topVW.rows;
table(topVW.rows as Record<string, unknown>[]);
// ======================================================================
// 4. SECURITY STATUS
// ======================================================================
header('4. SECURITY STATUS');
const secRows = await db.execute(sql`
SELECT
COALESCE(security_status, 'null') AS status,
COUNT(*)::int AS count,
ROUND(COUNT(*)::numeric / NULLIF(${total}, 0) * 100, 1) AS pct
FROM skills WHERE is_blocked = false
GROUP BY security_status ORDER BY count DESC
`);
report.securityDist = secRows.rows;
table(secRows.rows as Record<string, unknown>[]);
// ======================================================================
// 5. CONTENT ANALYSIS
// ======================================================================
header('5. CONTENT ANALYSIS');
const csRows = await db.execute(sql`
SELECT
COUNT(*) FILTER (WHERE raw_content IS NOT NULL)::int AS has_content,
COUNT(*) FILTER (WHERE raw_content IS NULL)::int AS no_content,
ROUND(AVG(LENGTH(raw_content)) FILTER (WHERE raw_content IS NOT NULL))::int AS avg_content_len,
MAX(LENGTH(raw_content))::int AS max_content_len,
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY LENGTH(raw_content))
FILTER (WHERE raw_content IS NOT NULL)::int AS median_content_len,
ROUND(AVG(LENGTH(description)))::int AS avg_desc_len,
COUNT(*) FILTER (WHERE LENGTH(description) > 100)::int AS good_desc,
COUNT(*) FILTER (WHERE LENGTH(description) <= 20)::int AS short_desc,
COUNT(*) FILTER (WHERE version IS NOT NULL)::int AS has_version,
COUNT(*) FILTER (WHERE license IS NOT NULL)::int AS has_license,
COUNT(*) FILTER (WHERE author IS NOT NULL)::int AS has_author,
COUNT(*) FILTER (WHERE homepage IS NOT NULL)::int AS has_homepage
FROM skills WHERE is_blocked = false
`);
const cs = csRows.rows[0] as Record<string, unknown>;
report.contentStats = cs;
console.log(` Has raw_content: ${num(cs.has_content)} (${pct(cs.has_content, total)})`);
console.log(` No raw_content: ${num(cs.no_content)} (${pct(cs.no_content, total)})`);
console.log(` Avg content length: ${num(cs.avg_content_len)} chars`);
console.log(` Median content len: ${num(cs.median_content_len)} chars`);
console.log(` Max content length: ${num(cs.max_content_len)} chars`);
console.log(` Avg desc length: ${num(cs.avg_desc_len)} chars`);
console.log(` Good desc (>100ch): ${num(cs.good_desc)} (${pct(cs.good_desc, total)})`);
console.log(` Short desc (<=20): ${num(cs.short_desc)} (${pct(cs.short_desc, total)})`);
console.log(` Has version: ${num(cs.has_version)} (${pct(cs.has_version, total)})`);
console.log(` Has license: ${num(cs.has_license)} (${pct(cs.has_license, total)})`);
console.log(` Has author: ${num(cs.has_author)} (${pct(cs.has_author, total)})`);
console.log(` Has homepage: ${num(cs.has_homepage)} (${pct(cs.has_homepage, total)})`);
subHeader('Content Length Distribution');
const clRows = await db.execute(sql`
SELECT
CASE
WHEN raw_content IS NULL THEN 'null'
WHEN LENGTH(raw_content) = 0 THEN 'empty'
WHEN LENGTH(raw_content) < 200 THEN '<200'
WHEN LENGTH(raw_content) < 500 THEN '200-500'
WHEN LENGTH(raw_content) < 1000 THEN '500-1K'
WHEN LENGTH(raw_content) < 3000 THEN '1K-3K'
WHEN LENGTH(raw_content) < 5000 THEN '3K-5K'
WHEN LENGTH(raw_content) < 10000 THEN '5K-10K'
WHEN LENGTH(raw_content) < 50000 THEN '10K-50K'
ELSE '50K+'
END AS range,
COUNT(*)::int AS skills
FROM skills WHERE is_blocked = false
GROUP BY 1 ORDER BY MIN(COALESCE(LENGTH(raw_content), -1))
`);
report.contentLenDist = clRows.rows;
table(clRows.rows as Record<string, unknown>[]);
// ======================================================================
// 6. TRIGGERS & COMPATIBILITY
// ======================================================================
header('6. TRIGGERS & COMPATIBILITY');
const trRows = await db.execute(sql`
SELECT
COUNT(*) FILTER (WHERE triggers IS NOT NULL)::int AS has_triggers,
COUNT(*) FILTER (WHERE triggers IS NULL)::int AS no_triggers,
COUNT(*) FILTER (WHERE triggers->>'filePatterns' IS NOT NULL
AND triggers->>'filePatterns' != '[]'
AND triggers->>'filePatterns' != 'null')::int AS has_file_patterns,
COUNT(*) FILTER (WHERE triggers->>'keywords' IS NOT NULL
AND triggers->>'keywords' != '[]'
AND triggers->>'keywords' != 'null')::int AS has_keywords,
COUNT(*) FILTER (WHERE triggers->>'languages' IS NOT NULL
AND triggers->>'languages' != '[]'
AND triggers->>'languages' != 'null')::int AS has_languages,
COUNT(*) FILTER (WHERE compatibility IS NOT NULL)::int AS has_compat,
COUNT(*) FILTER (WHERE compatibility->>'platforms' IS NOT NULL
AND compatibility->>'platforms' != '[]'
AND compatibility->>'platforms' != 'null')::int AS has_platforms,
COUNT(*) FILTER (WHERE compatibility->>'requires' IS NOT NULL
AND compatibility->>'requires' != '[]'
AND compatibility->>'requires' != 'null')::int AS has_requires
FROM skills WHERE is_blocked = false
`);
const tr = trRows.rows[0] as Record<string, unknown>;
report.triggerStats = tr;
console.log(` Has triggers: ${num(tr.has_triggers)} (${pct(tr.has_triggers, total)})`);
console.log(` No triggers: ${num(tr.no_triggers)} (${pct(tr.no_triggers, total)})`);
console.log(` - filePatterns: ${num(tr.has_file_patterns)}`);
console.log(` - keywords: ${num(tr.has_keywords)}`);
console.log(` - languages: ${num(tr.has_languages)}`);
console.log(` Has compatibility: ${num(tr.has_compat)}`);
console.log(` - platforms: ${num(tr.has_platforms)}`);
console.log(` - requires: ${num(tr.has_requires)}`);
// ======================================================================
// 7. FRESHNESS
// ======================================================================
header('7. FRESHNESS & ACTIVITY');
subHeader('Created At Distribution');
const crRows = await db.execute(sql`
SELECT
CASE
WHEN created_at > NOW() - INTERVAL '7 days' THEN 'Last 7d'
WHEN created_at > NOW() - INTERVAL '30 days' THEN 'Last 30d'
WHEN created_at > NOW() - INTERVAL '90 days' THEN 'Last 90d'
WHEN created_at > NOW() - INTERVAL '180 days' THEN 'Last 180d'
WHEN created_at > NOW() - INTERVAL '365 days' THEN 'Last 365d'
ELSE 'Older'
END AS range,
COUNT(*)::int AS skills
FROM skills WHERE is_blocked = false
GROUP BY 1 ORDER BY MIN(NOW() - created_at)
`);
report.createdDist = crRows.rows;
table(crRows.rows as Record<string, unknown>[]);
subHeader('Updated At Distribution');
const upRows = await db.execute(sql`
SELECT
CASE
WHEN updated_at > NOW() - INTERVAL '7 days' THEN 'Last 7d'
WHEN updated_at > NOW() - INTERVAL '30 days' THEN 'Last 30d'
WHEN updated_at > NOW() - INTERVAL '90 days' THEN 'Last 90d'
WHEN updated_at > NOW() - INTERVAL '180 days' THEN 'Last 180d'
WHEN updated_at > NOW() - INTERVAL '365 days' THEN 'Last 365d'
ELSE 'Older'
END AS range,
COUNT(*)::int AS skills
FROM skills WHERE is_blocked = false
GROUP BY 1 ORDER BY MIN(NOW() - updated_at)
`);
report.updatedDist = upRows.rows;
table(upRows.rows as Record<string, unknown>[]);
subHeader('Last Downloaded Distribution');
const ldRows = await db.execute(sql`
SELECT
CASE
WHEN last_downloaded_at IS NULL THEN 'Never'
WHEN last_downloaded_at > NOW() - INTERVAL '7 days' THEN 'Last 7d'
WHEN last_downloaded_at > NOW() - INTERVAL '30 days' THEN 'Last 30d'
WHEN last_downloaded_at > NOW() - INTERVAL '90 days' THEN 'Last 90d'
ELSE 'Older'
END AS range,
COUNT(*)::int AS skills
FROM skills WHERE is_blocked = false
GROUP BY 1 ORDER BY MIN(COALESCE(last_downloaded_at, '1970-01-01'::timestamp))
`);
report.lastDownDist = ldRows.rows;
table(ldRows.rows as Record<string, unknown>[]);
// ======================================================================
// 8. TOP OWNERS
// ======================================================================
header('8. TOP OWNERS');
subHeader('Top 30 Owners by Skill Count');
const owRows = await db.execute(sql`
SELECT
github_owner AS owner,
COUNT(*)::int AS skills,
COUNT(DISTINCT github_repo)::int AS repos,
MAX(github_stars)::int AS max_stars,
COALESCE(SUM(download_count), 0)::int AS downloads,
COALESCE(SUM(view_count), 0)::int AS views
FROM skills WHERE is_blocked = false
GROUP BY github_owner ORDER BY skills DESC LIMIT 30
`);
report.topOwnersByCount = owRows.rows;
table(owRows.rows as Record<string, unknown>[]);
subHeader('Top 20 Owners by Stars');
const osRows = await db.execute(sql`
SELECT
github_owner AS owner,
MAX(github_stars)::int AS max_stars,
COUNT(*)::int AS skills,
COUNT(DISTINCT github_repo)::int AS repos,
COALESCE(SUM(download_count), 0)::int AS downloads
FROM skills WHERE is_blocked = false
GROUP BY github_owner ORDER BY max_stars DESC LIMIT 20
`);
report.topOwnersByStars = osRows.rows;
table(osRows.rows as Record<string, unknown>[]);
subHeader('Owner Concentration');
const ocRows = await db.execute(sql`
WITH ranked AS (
SELECT github_owner, COUNT(*)::int AS cnt,
ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn
FROM skills WHERE is_blocked = false
GROUP BY github_owner
)
SELECT
SUM(cnt) FILTER (WHERE rn <= 1)::int AS top_1,
SUM(cnt) FILTER (WHERE rn <= 5)::int AS top_5,
SUM(cnt) FILTER (WHERE rn <= 10)::int AS top_10,
SUM(cnt) FILTER (WHERE rn <= 20)::int AS top_20,
SUM(cnt) FILTER (WHERE rn <= 50)::int AS top_50,
SUM(cnt)::int AS total,
COUNT(*)::int AS owners
FROM ranked
`);
const oc = ocRows.rows[0] as Record<string, unknown>;
report.ownerConcentration = oc;
console.log(` Top 1 owner: ${num(oc.top_1)} skills (${pct(oc.top_1, oc.total)})`);
console.log(` Top 5 owners: ${num(oc.top_5)} skills (${pct(oc.top_5, oc.total)})`);
console.log(` Top 10 owners: ${num(oc.top_10)} skills (${pct(oc.top_10, oc.total)})`);
console.log(` Top 20 owners: ${num(oc.top_20)} skills (${pct(oc.top_20, oc.total)})`);
console.log(` Top 50 owners: ${num(oc.top_50)} skills (${pct(oc.top_50, oc.total)})`);
console.log(` Total owners: ${num(oc.owners)}`);
// ======================================================================
// 9. MULTI-SKILL REPOS
// ======================================================================
header('9. MULTI-SKILL REPOS');
subHeader('Repos with 20+ Skills (Aggregator Candidates)');
const aggRows = await db.execute(sql`
SELECT
github_owner || '/' || github_repo AS repo,
COUNT(*)::int AS skills,
MAX(github_stars)::int AS stars,
COALESCE(SUM(download_count), 0)::int AS downloads
FROM skills WHERE is_blocked = false
GROUP BY github_owner, github_repo
HAVING COUNT(*) >= 20
ORDER BY skills DESC LIMIT 30
`);
report.aggregatorCandidates = aggRows.rows;
table(aggRows.rows as Record<string, unknown>[]);
subHeader('Repos with 3-19 Skills (Collection Candidates)');
const colRows = await db.execute(sql`
SELECT
github_owner || '/' || github_repo AS repo,
COUNT(*)::int AS skills,
MAX(github_stars)::int AS stars,
COALESCE(SUM(download_count), 0)::int AS downloads
FROM skills WHERE is_blocked = false
GROUP BY github_owner, github_repo
HAVING COUNT(*) BETWEEN 3 AND 19
ORDER BY skills DESC LIMIT 30
`);
report.collectionCandidates = colRows.rows;
table(colRows.rows as Record<string, unknown>[]);
subHeader('Skills-per-Repo Distribution');
const sprRows = await db.execute(sql`
WITH rc AS (
SELECT COUNT(*)::int AS cnt
FROM skills WHERE is_blocked = false
GROUP BY github_owner, github_repo
)
SELECT
CASE
WHEN cnt = 1 THEN '1 skill'
WHEN cnt = 2 THEN '2 skills'
WHEN cnt BETWEEN 3 AND 5 THEN '3-5'
WHEN cnt BETWEEN 6 AND 10 THEN '6-10'
WHEN cnt BETWEEN 11 AND 20 THEN '11-20'
WHEN cnt BETWEEN 21 AND 50 THEN '21-50'
WHEN cnt BETWEEN 51 AND 100 THEN '51-100'
ELSE '100+'
END AS per_repo,
COUNT(*)::int AS repos,
SUM(cnt)::int AS total_skills
FROM rc
GROUP BY 1 ORDER BY MIN(cnt)
`);
report.skillsPerRepo = sprRows.rows;
table(sprRows.rows as Record<string, unknown>[]);
// ======================================================================
// 10. CATEGORY DISTRIBUTION
// ======================================================================
header('10. CATEGORY DISTRIBUTION');
const catRows = await db.execute(sql`
SELECT c.name AS category, c.skill_count::int AS skills, c.id
FROM categories c
WHERE c.id NOT LIKE 'parent-%'
ORDER BY c.skill_count DESC
`);
report.categoryDist = catRows.rows;
table(catRows.rows as Record<string, unknown>[]);
const ncRows = await db.execute(sql`
SELECT COUNT(*)::int AS count
FROM skills s
WHERE s.is_blocked = false
AND NOT EXISTS (SELECT 1 FROM skill_categories sc WHERE sc.skill_id = s.id)
`);
report.uncategorizedCount = (ncRows.rows[0] as Record<string, unknown>)?.count;
console.log(`\n Skills with no category: ${num(report.uncategorizedCount)}`);
// ======================================================================
// 11. FLAGS & RATINGS
// ======================================================================
header('11. FLAGS & RATINGS');
const flRows = await db.execute(sql`
SELECT
COUNT(*) FILTER (WHERE is_verified = true)::int AS verified,
COUNT(*) FILTER (WHERE is_featured = true)::int AS featured,
COUNT(*) FILTER (WHERE rating IS NOT NULL)::int AS has_rating,
COUNT(*) FILTER (WHERE rating_count > 0)::int AS has_user_ratings,
ROUND(AVG(rating) FILTER (WHERE rating IS NOT NULL), 2)::numeric AS avg_rating,
MAX(rating_count)::int AS max_rating_count
FROM skills WHERE is_blocked = false
`);
const fl = flRows.rows[0] as Record<string, unknown>;
report.flags = fl;
console.log(` Verified: ${num(fl.verified)}`);
console.log(` Featured: ${num(fl.featured)}`);
console.log(` Has rating: ${num(fl.has_rating)}`);
console.log(` Has user ratings: ${num(fl.has_user_ratings)}`);
console.log(` Average rating: ${fl.avg_rating ?? 'N/A'}`);
console.log(` Max rating count: ${num(fl.max_rating_count)}`);
// ======================================================================
// 12. CROSS-TABLE STATS
// ======================================================================
header('12. CROSS-TABLE STATS');
const xtRows = await db.execute(sql`
SELECT
(SELECT COUNT(*)::int FROM users) AS users,
(SELECT COUNT(*)::int FROM ratings) AS ratings,
(SELECT COUNT(*)::int FROM installations) AS installations,
(SELECT COUNT(*)::int FROM favorites) AS favorites,
(SELECT COUNT(*)::int FROM discovered_repos) AS discovered_repos,
(SELECT COUNT(*) FILTER (WHERE has_skill_md = true)::int FROM discovered_repos) AS repos_with_skills,
(SELECT COUNT(*)::int FROM removal_requests) AS removal_requests,
(SELECT COUNT(*)::int FROM add_requests) AS add_requests,
(SELECT COUNT(*) FILTER (WHERE unsubscribed_at IS NULL)::int FROM email_subscriptions) AS subscribers
`);
const xt = xtRows.rows[0] as Record<string, unknown>;
report.crossStats = xt;
console.log(` Users: ${num(xt.users)}`);
console.log(` Ratings: ${num(xt.ratings)}`);
console.log(` Installations: ${num(xt.installations)}`);
console.log(` Favorites: ${num(xt.favorites)}`);
console.log(` Discovered repos: ${num(xt.discovered_repos)}`);
console.log(` - with skills: ${num(xt.repos_with_skills)}`);
console.log(` Removal requests: ${num(xt.removal_requests)}`);
console.log(` Add requests: ${num(xt.add_requests)}`);
console.log(` Subscribers: ${num(xt.subscribers)}`);
subHeader('Installations by Platform');
const ipRows = await db.execute(sql`
SELECT platform, COUNT(*)::int AS count
FROM installations GROUP BY platform ORDER BY count DESC
`);
report.installByPlatform = ipRows.rows;
table(ipRows.rows as Record<string, unknown>[]);
// ======================================================================
// 13. USABILITY SIGNALS
// ======================================================================
header('13. USABILITY SIGNALS FOR CURATION');
const usRows = await db.execute(sql`
SELECT
COUNT(*) FILTER (
WHERE LENGTH(description) > 50
AND raw_content IS NOT NULL AND LENGTH(raw_content) > 500
AND security_status = 'pass'
)::int AS strong_standalone,
COUNT(*) FILTER (
WHERE triggers IS NOT NULL AND triggers != '{}'::jsonb
AND (triggers->>'filePatterns' IS NOT NULL AND triggers->>'filePatterns' != '[]')
)::int AS has_file_triggers,
COUNT(*) FILTER (WHERE COALESCE(download_count, 0) > 0)::int AS ever_downloaded,
COUNT(*) FILTER (
WHERE github_stars >= 10
AND LENGTH(description) > 50
AND raw_content IS NOT NULL
AND security_status = 'pass'
)::int AS high_quality,
COUNT(*) FILTER (
WHERE github_stars >= 100
AND COALESCE(download_count, 0) > 0
AND security_status = 'pass'
)::int AS premium,
COUNT(*) FILTER (
WHERE source_format = 'skill.md'
AND LENGTH(description) > 50
AND raw_content IS NOT NULL AND LENGTH(raw_content) > 300
AND security_status = 'pass'
)::int AS skillmd_quality
FROM skills WHERE is_blocked = false
`);
const us = usRows.rows[0] as Record<string, unknown>;
report.usability = us;
console.log(` Strong standalone candidates: ${num(us.strong_standalone)}`);
console.log(` (desc>50 + content>500 + security=pass)`);
console.log(` With file triggers: ${num(us.has_file_triggers)}`);
console.log(` Ever downloaded: ${num(us.ever_downloaded)}`);
console.log(` High quality (stars>10+desc+sec): ${num(us.high_quality)}`);
console.log(` Premium (stars>100+dl>0+sec): ${num(us.premium)}`);
console.log(` SKILL.md with quality: ${num(us.skillmd_quality)}`);
// ======================================================================
// 14. STANDALONE vs PROJECT-BOUND
// ======================================================================
header('14. STANDALONE vs PROJECT-BOUND HEURISTIC');
subHeader('Single vs Multi-Skill Repos');
const smRows = await db.execute(sql`
WITH rc AS (
SELECT github_owner, github_repo, COUNT(*)::int AS cnt
FROM skills WHERE is_blocked = false
GROUP BY github_owner, github_repo
)
SELECT
COUNT(*) FILTER (WHERE cnt = 1)::int AS single,
COUNT(*) FILTER (WHERE cnt = 2)::int AS two,
COUNT(*) FILTER (WHERE cnt >= 3)::int AS multi,
COUNT(*)::int AS total
FROM rc
`);
const sm = smRows.rows[0] as Record<string, unknown>;
report.repoTypes = sm;
console.log(` Single-skill repos: ${num(sm.single)} (${pct(sm.single, sm.total)})`);
console.log(` Two-skill repos: ${num(sm.two)} (${pct(sm.two, sm.total)})`);
console.log(` Multi-skill repos: ${num(sm.multi)} (${pct(sm.multi, sm.total)})`);
subHeader('Name Pattern Signals');
const npRows = await db.execute(sql`
SELECT
CASE
WHEN name ILIKE '%rule%' OR name ILIKE '%config%' OR name ILIKE '%setup%' THEN 'rules/config/setup'
WHEN name ILIKE '%my-%' OR name ILIKE '%my_%' THEN 'starts with my-'
WHEN name ILIKE '%cursor%' OR name ILIKE '%claude%' OR name ILIKE '%copilot%' THEN 'tool-specific'
WHEN name ILIKE '%project%' OR name ILIKE '%team%' OR name ILIKE '%internal%' THEN 'project/team'
WHEN name ILIKE '%.mdc' OR name ILIKE '%cursorrule%' THEN 'cursor rules file'
ELSE 'generic/other'
END AS pattern,
COUNT(*)::int AS skills
FROM skills WHERE is_blocked = false
GROUP BY 1 ORDER BY skills DESC
`);
report.namePatterns = npRows.rows;
table(npRows.rows as Record<string, unknown>[]);
// ======================================================================
// 15. SAMPLE SKILLS
// ======================================================================
header('15. SAMPLE SKILLS');
subHeader('Top 15 by Stars');
const tsRows = await db.execute(sql`
SELECT id, name, github_stars AS stars, COALESCE(download_count,0) AS dl,
security_status AS sec, source_format AS fmt,
LEFT(description, 80) AS description
FROM skills WHERE is_blocked = false
ORDER BY github_stars DESC LIMIT 15
`);
report.topByStars = tsRows.rows;
table(tsRows.rows as Record<string, unknown>[]);
subHeader('Top 15 by Downloads (actually used)');
const tdRows = await db.execute(sql`
SELECT id, name, download_count AS dl, view_count AS views,
github_stars AS stars, security_status AS sec,
LEFT(description, 80) AS description
FROM skills WHERE is_blocked = false AND COALESCE(download_count,0) > 0
ORDER BY download_count DESC LIMIT 15
`);
report.topByDL = tdRows.rows;
table(tdRows.rows as Record<string, unknown>[]);
subHeader('SKILL.md + Stars>50 + Downloads>0');
const qsRows = await db.execute(sql`
SELECT id, name, github_stars AS stars, download_count AS dl,
security_status AS sec, LEFT(description, 80) AS description
FROM skills
WHERE is_blocked = false AND source_format = 'skill.md'
AND github_stars >= 50 AND COALESCE(download_count, 0) > 0
ORDER BY github_stars DESC LIMIT 20
`);
report.qualitySkillMd = qsRows.rows;
table(qsRows.rows as Record<string, unknown>[]);
// ======================================================================
// 16. DISCOVERED REPOS
// ======================================================================
header('16. DISCOVERED REPOS');
const drRows = await db.execute(sql`
SELECT
discovered_via AS source,
COUNT(*)::int AS total,
COUNT(*) FILTER (WHERE has_skill_md = true)::int AS with_skills,
COUNT(*) FILTER (WHERE last_scanned IS NOT NULL)::int AS scanned,
COUNT(*) FILTER (WHERE is_archived = true)::int AS archived
FROM discovered_repos
GROUP BY discovered_via ORDER BY total DESC
`);
report.discoveredStats = drRows.rows;
table(drRows.rows as Record<string, unknown>[]);
// ======================================================================
// 17. CACHED FILES
// ======================================================================
header('17. CACHED FILES');
const cfRows = await db.execute(sql`
SELECT
COUNT(*) FILTER (WHERE cached_files IS NOT NULL)::int AS has_cache,
COUNT(*) FILTER (WHERE cached_files IS NULL)::int AS no_cache
FROM skills WHERE is_blocked = false
`);
const cf = cfRows.rows[0] as Record<string, unknown>;
report.cacheStats = cf;
console.log(` Has cached files: ${num(cf.has_cache)} (${pct(cf.has_cache, total)})`);
console.log(` No cached files: ${num(cf.no_cache)} (${pct(cf.no_cache, total)})`);
// ======================================================================
// 19. BRANCH DISTRIBUTION
// ======================================================================
header('19. BRANCH DISTRIBUTION');
const brRows = await db.execute(sql`
SELECT COALESCE(branch, 'null') AS branch, COUNT(*)::int AS count
FROM skills WHERE is_blocked = false
GROUP BY branch ORDER BY count DESC LIMIT 10
`);
report.branchDist = brRows.rows;
table(brRows.rows as Record<string, unknown>[]);
// ======================================================================
// 20. DUPLICATES
// ======================================================================
header('20. POTENTIAL DUPLICATES');
subHeader('Same Name + Description');
const ddRows = await db.execute(sql`
SELECT name, LEFT(description, 60) AS desc, COUNT(*)::int AS occurrences,
STRING_AGG(DISTINCT github_owner, ', ') AS owners
FROM skills
WHERE is_blocked = false AND description IS NOT NULL AND LENGTH(description) > 10
GROUP BY name, description HAVING COUNT(*) > 1
ORDER BY occurrences DESC LIMIT 15
`);
report.exactDupes = ddRows.rows;
table(ddRows.rows as Record<string, unknown>[]);
subHeader('Same Content Hash (identical content, 3+ copies)');
const chRows = await db.execute(sql`
SELECT content_hash, COUNT(*)::int AS copies,
MIN(name) AS sample_name,
STRING_AGG(DISTINCT github_owner, ', ') AS owners
FROM skills
WHERE is_blocked = false AND content_hash IS NOT NULL
GROUP BY content_hash HAVING COUNT(*) > 2
ORDER BY copies DESC LIMIT 15
`);
report.hashDupes = chRows.rows;
table(chRows.rows as Record<string, unknown>[]);
// ======================================================================
// DONE
// ======================================================================
const duration = ((Date.now() - startTime) / 1000).toFixed(1);
header(`EXPLORATION COMPLETE (${duration}s)`);
console.log(`
KEY QUESTIONS TO ANSWER:
1. What % of skills are SKILL.md vs other formats?
2. What % have been downloaded at least once? (real usage)
3. How concentrated is ownership? (top 10 owners = ?%)
4. How many multi-skill repos? (collection/aggregator candidates)
5. How many pass security + have good content? (curation-ready)
6. How many duplicates exist?
`);
// Save JSON report
report.generatedAt = new Date().toISOString();
report.durationSeconds = parseFloat(duration);
const scriptDir = dirname(fileURLToPath(import.meta.url));
const reportPath = resolve(scriptDir, 'explore-report.json');
writeFileSync(reportPath, JSON.stringify(report, null, 2), 'utf-8');
console.log(` Full report saved to: ${reportPath}`);
await closeDb();
}
// ─── Run ───────────────────────────────────────────────────────────────────────
runExploration().catch(async (err) => {
console.error('Exploration failed:', err);
await closeDb();
process.exit(1);
});

14
scripts/init-db.sql
View File

@@ -348,6 +348,7 @@ BEGIN
-- Excluded: cached_files (download cache, not content change)
-- Excluded: indexed_at, last_scanned (indexer bookkeeping)
-- Excluded: github_stars, github_forks (popularity metrics, not content)
-- Excluded: quality_score, quality_details, skill_type, is_duplicate, canonical_skill_id, repo_skill_count (curation metadata)
IF ROW(NEW.name, NEW.description, NEW.github_owner, NEW.github_repo, NEW.skill_path,
NEW.branch, NEW.commit_sha, NEW.source_format, NEW.version, NEW.license, NEW.author, NEW.homepage,
NEW.compatibility, NEW.triggers,
@@ -461,6 +462,19 @@ CREATE INDEX IF NOT EXISTS idx_skills_source_format ON skills(source_format);
ALTER TABLE skills ADD COLUMN IF NOT EXISTS last_downloaded_at TIMESTAMP WITH TIME ZONE;
CREATE INDEX IF NOT EXISTS idx_skills_last_downloaded ON skills(last_downloaded_at DESC NULLS LAST);
-- Curation columns (Phase 2 - February 2026)
ALTER TABLE skills ADD COLUMN IF NOT EXISTS quality_score INTEGER;
ALTER TABLE skills ADD COLUMN IF NOT EXISTS quality_details JSONB;
ALTER TABLE skills ADD COLUMN IF NOT EXISTS skill_type TEXT; -- 'standalone', 'project-bound', 'collection', 'aggregator'
ALTER TABLE skills ADD COLUMN IF NOT EXISTS is_duplicate BOOLEAN DEFAULT FALSE;
ALTER TABLE skills ADD COLUMN IF NOT EXISTS canonical_skill_id TEXT; -- points to original if duplicate
ALTER TABLE skills ADD COLUMN IF NOT EXISTS repo_skill_count INTEGER; -- cached count of skills in repo
CREATE INDEX IF NOT EXISTS idx_skills_quality ON skills(quality_score DESC NULLS LAST);
CREATE INDEX IF NOT EXISTS idx_skills_type ON skills(skill_type);
CREATE INDEX IF NOT EXISTS idx_skills_duplicate ON skills(is_duplicate);
CREATE INDEX IF NOT EXISTS idx_skills_content_hash ON skills(content_hash);
-- Grant permissions
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO postgres;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO postgres;

1
services/indexer/src/skill-indexer.ts
View File

@@ -90,6 +90,7 @@ export async function indexSkill(
githubStars: content.repoMeta.stars,
githubForks: content.repoMeta.forks,
securityScore: analysis.security.score,
securityStatus: analysis.security.status,
contentHash: analysis.meta.contentHash,
rawContent: content.skillMd,
indexedAt: new Date(),