From caca09fbe728a6ea89f494b45e802043c466719a Mon Sep 17 00:00:00 2001 From: airano Date: Thu, 19 Feb 2026 14:31:25 +0330 Subject: [PATCH] =?UTF-8?q?feat(curation):=20data=20quality=20pipeline=20?= =?UTF-8?q?=E2=80=94=20Phases=201-3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add comprehensive data curation system to clean up the 197K skill dataset and show only quality browse-ready skills to users. Phase 1 — Database exploration: - Explore scripts (explore.ts, explore.mjs, explore.sql) for analysis - Discovered: 69% duplicates, 77% aggregator/fork noise Phase 2 — Data cleanup and classification: - Schema: 6 new curation columns + 4 indexes - curate.mjs: 8-step pipeline (classify, dedup, fork detection, etc.) - Result: 197K → 60K unique → 16K browse-ready skills - Bug fix: securityStatus was computed but never stored during crawl Phase 3 — UI browse-ready filters: - browseReadyFilter applied to 17+ query functions - Homepage stats show accurate browse-ready counts - Stats API filtered (previously had no WHERE clause) - Category counts recalculated (e.g. 45K → 3.1K) - Featured skills exclude duplicates and aggregators Co-Authored-By: Claude Opus 4.6 --- apps/web/app/[locale]/page.tsx | 17 +- apps/web/app/api/stats/route.test.ts | 21 +- apps/web/app/api/stats/route.ts | 8 +- packages/db/src/queries.ts | 42 +- packages/db/src/schema.ts | 17 + scripts/curation/curate.mjs | 505 +++++++++++++++ scripts/curation/curate.sql | 214 ++++++ scripts/curation/explore.mjs | 652 +++++++++++++++++++ scripts/curation/explore.sql | 258 ++++++++ scripts/curation/explore.ts | 894 ++++++++++++++++++++++++++ scripts/init-db.sql | 14 + services/indexer/src/skill-indexer.ts | 1 + 12 files changed, 2615 insertions(+), 28 deletions(-) create mode 100644 scripts/curation/curate.mjs create mode 100644 scripts/curation/curate.sql create mode 100644 scripts/curation/explore.mjs create mode 100644 scripts/curation/explore.sql create mode 100644 scripts/curation/explore.ts diff --git a/apps/web/app/[locale]/page.tsx b/apps/web/app/[locale]/page.tsx index d3d2c5a..beca19d 100644 --- a/apps/web/app/[locale]/page.tsx +++ b/apps/web/app/[locale]/page.tsx @@ -16,27 +16,32 @@ async function getStats() { try { const db = createDb(); - // Get total skills count (SKILL.md only - real reusable skills) + // Browse-ready filter: exclude duplicates and aggregators + const browseReady = sql`${skills.isDuplicate} = false AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`; + + // Get total skills count (browse-ready, SKILL.md only) const skillsResult = await db .select({ count: sql`count(*)::int` }) .from(skills) - .where(sql`${skills.sourceFormat} = 'skill.md' AND ${skills.isBlocked} = false`); + .where(sql`${skills.sourceFormat} = 'skill.md' AND ${skills.isBlocked} = false AND ${browseReady}`); const totalSkills = skillsResult[0]?.count ?? 0; - // Get total downloads + // Get total downloads (browse-ready only) const downloadsResult = await db .select({ sum: sql`coalesce(sum(${skills.downloadCount}), 0)::int` }) - .from(skills); + .from(skills) + .where(browseReady); const totalDownloads = downloadsResult[0]?.sum ?? 0; // Get total categories const categories = await categoryQueries.getAll(db); const totalCategories = categories.length; - // Get unique contributors (github owners) + // Get unique contributors (browse-ready skills only) const contributorsResult = await db .select({ count: sql`count(distinct ${skills.githubOwner})::int` }) - .from(skills); + .from(skills) + .where(browseReady); const totalContributors = contributorsResult[0]?.count ?? 0; return { diff --git a/apps/web/app/api/stats/route.test.ts b/apps/web/app/api/stats/route.test.ts index 13c9086..15e7020 100644 --- a/apps/web/app/api/stats/route.test.ts +++ b/apps/web/app/api/stats/route.test.ts @@ -18,17 +18,26 @@ vi.mock('@/lib/cache', () => ({ // Mock the db module - must be before imports that use it vi.mock('@skillhub/db', () => { + const mockStatsRow = [{ totalSkills: 100, totalDownloads: 5000, totalContributors: 50 }]; + const mockCategoryRow = [{ count: 8 }]; return { createDb: vi.fn(() => ({ select: vi.fn().mockReturnValue({ - from: vi.fn().mockResolvedValue([ - { totalSkills: 100, totalDownloads: 5000, totalContributors: 50 }, - ]), + from: vi.fn((table: unknown) => { + // categories table query returns count directly (no .where()) + if (table === 'categories-table') { + return Promise.resolve(mockCategoryRow); + } + // skills table query has .where() chained + return { + where: vi.fn().mockResolvedValue(mockStatsRow), + }; + }), }), })), - skills: { downloadCount: 'download_count', githubOwner: 'github_owner' }, - categories: {}, - sql: vi.fn(() => 'mock-sql'), + skills: { downloadCount: 'download_count', githubOwner: 'github_owner', isDuplicate: 'is_duplicate', skillType: 'skill_type' }, + categories: 'categories-table', + sql: vi.fn((..._args: unknown[]) => 'mock-sql'), }; }); diff --git a/apps/web/app/api/stats/route.ts b/apps/web/app/api/stats/route.ts index e86dd84..cf8f599 100644 --- a/apps/web/app/api/stats/route.ts +++ b/apps/web/app/api/stats/route.ts @@ -34,14 +34,18 @@ export async function GET(request: NextRequest) { }); } - // Consolidate all skill stats into a single query for better performance + // Browse-ready filter: exclude duplicates and aggregators + const browseReady = sql`${skills.isDuplicate} = false AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`; + + // Consolidate all skill stats into a single query (browse-ready only) const statsResult = await db .select({ totalSkills: sql`count(*)::int`, totalDownloads: sql`coalesce(sum(${skills.downloadCount}), 0)::int`, totalContributors: sql`count(distinct ${skills.githubOwner})::int`, }) - .from(skills); + .from(skills) + .where(browseReady); // Get category count in separate query (different table) const categoryResult = await db diff --git a/packages/db/src/queries.ts b/packages/db/src/queries.ts index 3ac267e..8d495c2 100644 --- a/packages/db/src/queries.ts +++ b/packages/db/src/queries.ts @@ -30,6 +30,13 @@ function getRawClientLocal() { type DB = PostgresJsDatabase; +/** + * Curation filter: excludes duplicates and aggregator-only skills. + * Skills with skill_type=NULL (newly crawled, not yet curated) are included + * so they don't disappear between curate.mjs runs. + */ +const browseReadyFilter = sql`(${skills.isDuplicate} = false) AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`; + /** * Skill queries */ @@ -76,7 +83,8 @@ export const skillQueries = { } = options; const conditions = [ - eq(skills.isBlocked, false), // Filter out blocked skills + eq(skills.isBlocked, false), + browseReadyFilter, ]; // Filter by source format (default: SKILL.md only; 'all' shows everything) @@ -200,7 +208,8 @@ export const skillQueries = { const { query, category, platform, sourceFormat = 'skill.md', minStars = 0, minSecurity, verified } = options; const conditions = [ - eq(skills.isBlocked, false), // Filter out blocked skills + eq(skills.isBlocked, false), + browseReadyFilter, ]; // Filter by source format (default: SKILL.md only; 'all' shows everything) @@ -277,7 +286,7 @@ export const skillQueries = { return db .select() .from(skills) - .where(and(eq(skills.isFeatured, true), eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'))) + .where(and(eq(skills.isFeatured, true), eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'), browseReadyFilter)) .orderBy(desc(skills.githubStars)) .limit(limit) .offset(offset); @@ -290,7 +299,7 @@ export const skillQueries = { const result = await db .select({ count: sql`cast(count(*) as int)` }) .from(skills) - .where(and(eq(skills.isFeatured, true), eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'))); + .where(and(eq(skills.isFeatured, true), eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'), browseReadyFilter)); return result[0]?.count ?? 0; }, @@ -298,14 +307,14 @@ export const skillQueries = { * Get trending skills (most downloads in recent period) */ getTrending: async (db: DB, limit = 10) => { - return db.select().from(skills).where(and(eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'))).orderBy(desc(skills.downloadCount)).limit(limit); + return db.select().from(skills).where(and(eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'), browseReadyFilter)).orderBy(desc(skills.downloadCount)).limit(limit); }, /** * Get recently added skills with pagination (sorted by creation date) */ getRecent: async (db: DB, limit = 10, offset = 0) => { - return db.select().from(skills).where(and(eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'))).orderBy(desc(skills.createdAt)).limit(limit).offset(offset); + return db.select().from(skills).where(and(eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'), browseReadyFilter)).orderBy(desc(skills.createdAt)).limit(limit).offset(offset); }, /** @@ -320,6 +329,7 @@ export const skillQueries = { and( eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'), + browseReadyFilter, gte(skills.createdAt, sevenDaysAgo) ) ) @@ -340,6 +350,7 @@ export const skillQueries = { and( eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'), + browseReadyFilter, gte(skills.createdAt, sevenDaysAgo) ) ); @@ -358,6 +369,7 @@ export const skillQueries = { and( eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'), + browseReadyFilter, sql`${skills.createdAt} < ${sevenDaysAgo}`, gte(skills.updatedAt, sevenDaysAgo), sql`${skills.updatedAt} > ${skills.createdAt} + interval '1 hour'` @@ -380,6 +392,7 @@ export const skillQueries = { and( eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'), + browseReadyFilter, sql`${skills.createdAt} < ${sevenDaysAgo}`, gte(skills.updatedAt, sevenDaysAgo), sql`${skills.updatedAt} > ${skills.createdAt} + interval '1 hour'` @@ -395,7 +408,7 @@ export const skillQueries = { const result = await db .select({ count: sql`cast(count(*) as int)` }) .from(skills) - .where(and(eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'))); + .where(and(eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'), browseReadyFilter)); return result[0]?.count ?? 0; }, @@ -410,7 +423,7 @@ export const skillQueries = { githubOwner: skills.githubOwner, }) .from(skills) - .where(and(eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'))); + .where(and(eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'), browseReadyFilter)); }, /** @@ -467,7 +480,7 @@ export const skillQueries = { return db .select() .from(skills) - .where(and(eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'))) + .where(and(eq(skills.isBlocked, false), eq(skills.sourceFormat, 'skill.md'), browseReadyFilter)) .orderBy( desc( sql`( @@ -580,6 +593,7 @@ export const skillQueries = { githubStars: skill.githubStars, githubForks: skill.githubForks, securityScore: skill.securityScore, + securityStatus: skill.securityStatus, sourceFormat: skill.sourceFormat, contentHash: skill.contentHash, rawContent: skill.rawContent, @@ -751,7 +765,7 @@ export const skillQueries = { ) => { const { limit = 24, offset = 0, sortBy = 'popularity', repo } = options; - const conditions = [eq(skills.githubOwner, owner), eq(skills.isBlocked, false)]; + const conditions = [eq(skills.githubOwner, owner), eq(skills.isBlocked, false), browseReadyFilter]; if (repo) conditions.push(eq(skills.githubRepo, repo)); const ownerCondition = and(...conditions); @@ -828,7 +842,7 @@ export const skillQueries = { skillCount: sql`count(*) OVER (PARTITION BY ${skills.githubRepo})::int`, }) .from(skills) - .where(and(eq(skills.githubOwner, owner), eq(skills.isBlocked, false))) + .where(and(eq(skills.githubOwner, owner), eq(skills.isBlocked, false), browseReadyFilter)) .orderBy(skills.githubRepo, desc(skills.githubStars)); }, @@ -836,7 +850,7 @@ export const skillQueries = { * Count skills by owner (for pagination), optionally filtered by repo */ countByOwner: async (db: DB, owner: string, repo?: string): Promise => { - const conditions = [eq(skills.githubOwner, owner), eq(skills.isBlocked, false)]; + const conditions = [eq(skills.githubOwner, owner), eq(skills.isBlocked, false), browseReadyFilter]; if (repo) conditions.push(eq(skills.githubRepo, repo)); const result = await db .select({ count: sql`cast(count(*) as int)` }) @@ -858,7 +872,7 @@ export const skillQueries = { maxStars: sql`COALESCE(MAX(${skills.githubStars}), 0)::int`, }) .from(skills) - .where(and(eq(skills.githubOwner, owner), eq(skills.isBlocked, false))); + .where(and(eq(skills.githubOwner, owner), eq(skills.isBlocked, false), browseReadyFilter)); return result[0] ?? { totalSkills: 0, totalDownloads: 0, totalViews: 0, totalRepos: 0, maxStars: 0 }; }, @@ -891,7 +905,7 @@ export const categoryQueries = { .select({ skill: skills }) .from(skillCategories) .innerJoin(skills, eq(skillCategories.skillId, skills.id)) - .where(eq(skillCategories.categoryId, categoryId)) + .where(and(eq(skillCategories.categoryId, categoryId), sql`${browseReadyFilter}`)) .orderBy(desc(skills.githubStars)) .limit(limit) .offset(offset); diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index b788a65..396237d 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -67,6 +67,19 @@ export const skills = pgTable( isBlocked: boolean('is_blocked').default(false), // Blocked from re-indexing (owner requested removal) lastScanned: timestamp('last_scanned'), + // Curation (populated by batch scripts, not crawler) + qualityScore: integer('quality_score'), // 0-100 from analyzer + qualityDetails: jsonb('quality_details').$type<{ + documentation: number; + maintenance: number; + popularity: number; + factors: Array<{ name: string; score: number; weight: number; details?: string }>; + }>(), + skillType: text('skill_type').$type<'standalone' | 'project-bound' | 'collection' | 'aggregator'>(), + isDuplicate: boolean('is_duplicate').default(false), + canonicalSkillId: text('canonical_skill_id'), // points to the "original" if this is a duplicate + repoSkillCount: integer('repo_skill_count'), // cached count of skills in same repo + // Content (cached) contentHash: text('content_hash'), rawContent: text('raw_content'), @@ -103,6 +116,10 @@ export const skills = pgTable( updatedIdx: index('idx_skills_updated').on(table.updatedAt), sourceFormatIdx: index('idx_skills_source_format').on(table.sourceFormat), lastDownloadedIdx: index('idx_skills_last_downloaded').on(table.lastDownloadedAt), + qualityIdx: index('idx_skills_quality').on(table.qualityScore), + skillTypeIdx: index('idx_skills_type').on(table.skillType), + duplicateIdx: index('idx_skills_duplicate').on(table.isDuplicate), + contentHashIdx: index('idx_skills_content_hash').on(table.contentHash), }) ); diff --git a/scripts/curation/curate.mjs b/scripts/curation/curate.mjs new file mode 100644 index 0000000..b2e1640 --- /dev/null +++ b/scripts/curation/curate.mjs @@ -0,0 +1,505 @@ +#!/usr/bin/env node +/** + * Phase 2: Data Cleanup & Classification + * + * Runs all curation steps in order: + * Step 1: Compute repo_skill_count for every skill + * Step 2: Mark aggregators (repos with 50+ skills) + * Step 3: Classify remaining repos (collection / standalone / project-bound) + * Step 4: Fill missing content_hash values + * Step 5: Mark duplicates by content_hash (canonical = highest stars or oldest) + * Step 6: Detect fork marketplace repos + * Step 7: Recalculate category skill counts (browse-ready only) + * Step 8: Summary report + * + * Usage: + * DATABASE_URL=postgres://... node scripts/curation/curate.mjs + * # Or with local Docker: + * DATABASE_URL=postgresql://postgres:postgres@localhost:5432/skillhub node scripts/curation/curate.mjs + * + * Options: + * --dry-run Show what would change without writing + * --step=N Run only step N (1-8) + */ + +import { createRequire } from 'module'; +import { resolve, dirname } from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +// ─── Find pg module ─── +let pg; +const tryPaths = [ + resolve(__dirname, '../..', 'package.json'), + '/tmp/package.json', + process.env.APPDATA ? resolve(process.env.APPDATA, '..', 'Local', 'Temp', 'package.json') : null, +].filter(Boolean); +for (const p of tryPaths) { + try { pg = createRequire(p)('pg'); break; } catch {} +} +if (!pg) { console.error('pg not found. Run: npm install pg'); process.exit(1); } + +// ─── Config ─── +const DATABASE_URL = process.env.DATABASE_URL || 'postgresql://postgres:postgres@localhost:5432/skillhub'; +const DRY_RUN = process.argv.includes('--dry-run'); +const stepArg = process.argv.find(a => a.startsWith('--step=')); +const ONLY_STEP = stepArg ? parseInt(stepArg.split('=')[1]) : null; + +// ─── Helpers ─── +function header(step, title) { + console.log(`\n${'='.repeat(70)}\n STEP ${step}: ${title}\n${'='.repeat(70)}`); +} + +const n = v => Number(v ?? 0).toLocaleString('en-US'); + +// ─── Database ─── +let client; + +async function connect() { + client = new pg.Client({ + connectionString: DATABASE_URL, + ssl: false, + connectionTimeoutMillis: 15000, + query_timeout: 600000, // 10 min for big updates + keepAlive: true, + }); + await client.connect(); + console.log('Connected to database'); +} + +async function query(sql, params = []) { + const result = await client.query(sql, params); + return result; +} + +// ─── Step 1: Compute repo_skill_count ─── +async function step1_repoSkillCount() { + header(1, 'COMPUTE repo_skill_count'); + + const countResult = await query(` + SELECT COUNT(*)::int AS total + FROM skills + WHERE is_blocked = false AND repo_skill_count IS NULL + `); + console.log(` Skills without repo_skill_count: ${n(countResult.rows[0].total)}`); + + if (DRY_RUN) { console.log(' [DRY RUN] Would update all skills'); return; } + + const result = await query(` + UPDATE skills s SET repo_skill_count = sub.cnt + FROM ( + SELECT github_owner, github_repo, COUNT(*)::int AS cnt + FROM skills WHERE is_blocked = false + GROUP BY github_owner, github_repo + ) sub + WHERE s.github_owner = sub.github_owner + AND s.github_repo = sub.github_repo + AND s.is_blocked = false + `); + console.log(` Updated: ${n(result.rowCount)} skills`); +} + +// ─── Step 2: Mark aggregators ─── +async function step2_markAggregators() { + header(2, 'MARK AGGREGATORS (repos with 50+ skills)'); + + // Preview + const preview = await query(` + SELECT github_owner || '/' || github_repo AS repo, COUNT(*)::int AS skills, + MAX(github_stars)::int AS stars + FROM skills WHERE is_blocked = false + GROUP BY github_owner, github_repo + HAVING COUNT(*) >= 50 + ORDER BY skills DESC LIMIT 20 + `); + console.log(` Top aggregator repos (${preview.rowCount} total):`); + for (const r of preview.rows.slice(0, 10)) { + console.log(` ${r.repo}: ${n(r.skills)} skills (${n(r.stars)} stars)`); + } + + if (DRY_RUN) { console.log(' [DRY RUN] Would mark skills in these repos'); return; } + + const result = await query(` + UPDATE skills s SET skill_type = 'aggregator' + FROM ( + SELECT github_owner, github_repo + FROM skills WHERE is_blocked = false + GROUP BY github_owner, github_repo + HAVING COUNT(*) >= 50 + ) agg + WHERE s.github_owner = agg.github_owner + AND s.github_repo = agg.github_repo + AND s.is_blocked = false + AND s.skill_type IS NULL + `); + console.log(` Marked as aggregator: ${n(result.rowCount)} skills`); +} + +// ─── Step 3: Classify remaining repos ─── +async function step3_classifyRemaining() { + header(3, 'CLASSIFY REMAINING REPOS'); + + // 3a: repos with 10-49 skills, name contains marketplace/awesome/collection → aggregator + if (!DRY_RUN) { + const aggByName = await query(` + UPDATE skills s SET skill_type = 'aggregator' + FROM ( + SELECT github_owner, github_repo + FROM skills WHERE is_blocked = false AND skill_type IS NULL + GROUP BY github_owner, github_repo + HAVING COUNT(*) >= 10 + AND (github_repo ILIKE '%marketplace%' + OR github_repo ILIKE '%awesome%' + OR github_repo ILIKE '%collection%' + OR github_repo ILIKE '%registry%') + ) agg + WHERE s.github_owner = agg.github_owner + AND s.github_repo = agg.github_repo + AND s.is_blocked = false + AND s.skill_type IS NULL + `); + console.log(` Aggregator by name (10+ & marketplace/awesome/registry): ${n(aggByName.rowCount)}`); + } + + // 3b: repos with 3-49 skills → collection + if (!DRY_RUN) { + const collections = await query(` + UPDATE skills s SET skill_type = 'collection' + FROM ( + SELECT github_owner, github_repo + FROM skills WHERE is_blocked = false AND skill_type IS NULL + GROUP BY github_owner, github_repo + HAVING COUNT(*) >= 3 + ) col + WHERE s.github_owner = col.github_owner + AND s.github_repo = col.github_repo + AND s.is_blocked = false + AND s.skill_type IS NULL + `); + console.log(` Collection (3+ skills in repo): ${n(collections.rowCount)}`); + } + + // 3c: single/two-skill repos with project-bound name patterns → project-bound + if (!DRY_RUN) { + const projectBound = await query(` + UPDATE skills SET skill_type = 'project-bound' + WHERE is_blocked = false AND skill_type IS NULL + AND repo_skill_count <= 2 + AND (name ILIKE '%my-%' OR name ILIKE '%my\\_%' ESCAPE '\\' + OR name ILIKE '%project%' OR name ILIKE '%team%' OR name ILIKE '%internal%' + OR name ILIKE '%.mdc' OR name ILIKE '%cursorrule%' + OR name ILIKE '%config%' OR name ILIKE '%setup%') + `); + console.log(` Project-bound (name pattern): ${n(projectBound.rowCount)}`); + } + + // 3d: remaining → standalone + if (!DRY_RUN) { + const standalone = await query(` + UPDATE skills SET skill_type = 'standalone' + WHERE is_blocked = false AND skill_type IS NULL + `); + console.log(` Standalone (remaining): ${n(standalone.rowCount)}`); + } + + // Summary + const summary = await query(` + SELECT skill_type, COUNT(*)::int AS count + FROM skills WHERE is_blocked = false + GROUP BY skill_type ORDER BY count DESC + `); + console.log('\n Classification summary:'); + for (const r of summary.rows) { + console.log(` ${(r.skill_type || 'null').padEnd(15)} ${n(r.count)}`); + } +} + +// ─── Step 4: Fill missing content_hash ─── +async function step4_fillContentHash() { + header(4, 'FILL MISSING content_hash'); + + const countResult = await query(` + SELECT COUNT(*)::int AS total + FROM skills WHERE is_blocked = false AND content_hash IS NULL AND raw_content IS NOT NULL + `); + const missing = countResult.rows[0].total; + console.log(` Skills missing content_hash: ${n(missing)}`); + + if (missing === 0) { console.log(' Nothing to do'); return; } + if (DRY_RUN) { console.log(' [DRY RUN] Would compute hash for these skills'); return; } + + // Use md5 as a reliable hash function available in PostgreSQL + const result = await query(` + UPDATE skills SET content_hash = md5(raw_content) + WHERE is_blocked = false AND content_hash IS NULL AND raw_content IS NOT NULL + `); + console.log(` Computed content_hash (md5) for: ${n(result.rowCount)} skills`); + + // Note: existing hashes use a JS numeric hash (analyzer.ts hashContent). + // We now use md5 for missing ones. For dedup purposes, same content → same hash + // regardless of algorithm, since we compare within hash groups. + // But mixed algorithms mean same content could have different hashes. + // Solution: re-hash ALL with md5 for consistency. + console.log(' Re-hashing ALL skills with md5 for consistency...'); + const rehash = await query(` + UPDATE skills SET content_hash = md5(raw_content) + WHERE is_blocked = false AND raw_content IS NOT NULL + `); + console.log(` Re-hashed: ${n(rehash.rowCount)} skills`); +} + +// ─── Step 5: Mark duplicates ─── +async function step5_markDuplicates() { + header(5, 'MARK DUPLICATES BY content_hash'); + + // Find duplicate groups + const dupGroups = await query(` + SELECT content_hash, COUNT(*)::int AS copies + FROM skills + WHERE is_blocked = false AND content_hash IS NOT NULL AND is_duplicate = false + GROUP BY content_hash + HAVING COUNT(*) > 1 + ORDER BY copies DESC + `); + const totalDupGroups = dupGroups.rowCount; + const totalDupSkills = dupGroups.rows.reduce((sum, r) => sum + r.copies, 0); + const removable = dupGroups.rows.reduce((sum, r) => sum + r.copies - 1, 0); + + console.log(` Duplicate groups: ${n(totalDupGroups)}`); + console.log(` Total skills in dup groups: ${n(totalDupSkills)}`); + console.log(` Removable (keeping 1 per group): ${n(removable)}`); + console.log(` Top 5 by copies:`); + for (const r of dupGroups.rows.slice(0, 5)) { + console.log(` hash=${r.content_hash}: ${r.copies} copies`); + } + + if (DRY_RUN) { console.log(' [DRY RUN] Would mark duplicates'); return; } + + // For each duplicate group: + // - canonical = highest github_stars, then oldest created_at + // - rest = is_duplicate = true, canonical_skill_id = canonical.id + const markResult = await query(` + WITH ranked AS ( + SELECT id, content_hash, + ROW_NUMBER() OVER ( + PARTITION BY content_hash + ORDER BY github_stars DESC NULLS LAST, + created_at ASC + ) AS rn + FROM skills + WHERE is_blocked = false AND content_hash IS NOT NULL AND is_duplicate = false + ), + canonicals AS ( + SELECT content_hash, id AS canonical_id + FROM ranked WHERE rn = 1 + ) + UPDATE skills s + SET is_duplicate = true, + canonical_skill_id = c.canonical_id + FROM ranked r + JOIN canonicals c ON r.content_hash = c.content_hash + WHERE s.id = r.id + AND r.rn > 1 + `); + console.log(` Marked as duplicate: ${n(markResult.rowCount)} skills`); + + // Verify + const verify = await query(` + SELECT + COUNT(*) FILTER (WHERE is_duplicate = false)::int AS unique_skills, + COUNT(*) FILTER (WHERE is_duplicate = true)::int AS duplicates + FROM skills WHERE is_blocked = false + `); + console.log(` After dedup: ${n(verify.rows[0].unique_skills)} unique, ${n(verify.rows[0].duplicates)} duplicates`); +} + +// ─── Step 6: Detect fork marketplace repos ─── +async function step6_detectForks() { + header(6, 'DETECT FORK MARKETPLACE REPOS'); + + // Known fork patterns: repos with same name across many owners + const forkPatterns = await query(` + SELECT github_repo, COUNT(DISTINCT github_owner)::int AS owners, + COUNT(*)::int AS total_skills + FROM skills + WHERE is_blocked = false + AND repo_skill_count >= 20 + GROUP BY github_repo + HAVING COUNT(DISTINCT github_owner) >= 3 + ORDER BY owners DESC + `); + + console.log(` Repo names appearing in 3+ owners (with 20+ skills each):`); + for (const r of forkPatterns.rows) { + console.log(` ${r.github_repo}: ${r.owners} owners, ${n(r.total_skills)} skills`); + } + + if (DRY_RUN) { console.log(' [DRY RUN] Would mark fork repo skills'); return; } + + // For fork repos, all copies should be checked as duplicates + // The canonical is already handled by step 5 (content_hash dedup) + // Here we just ensure they're typed correctly as aggregator + if (forkPatterns.rows.length > 0) { + const repoNames = forkPatterns.rows.map(r => r.github_repo); + const placeholders = repoNames.map((_, i) => `$${i + 1}`).join(','); + const markForks = await query(` + UPDATE skills SET skill_type = 'aggregator' + WHERE is_blocked = false + AND github_repo IN (${placeholders}) + AND repo_skill_count >= 20 + AND (skill_type IS NULL OR skill_type != 'aggregator') + `, repoNames); + console.log(` Re-classified as aggregator (fork repos): ${n(markForks.rowCount)}`); + } +} + +// ─── Step 7: Recalculate category counts ─── +async function step7_categoryCounts() { + header(7, 'RECALCULATE CATEGORY COUNTS'); + + // Update skill_count on each category to only count browse-ready skills + const result = await query(` + UPDATE categories c + SET skill_count = sub.cnt + FROM ( + SELECT sc.category_id, COUNT(*)::int AS cnt + FROM skill_categories sc + JOIN skills s ON sc.skill_id = s.id + WHERE s.is_blocked = false + AND s.is_duplicate = false + AND (s.skill_type IS NULL OR s.skill_type != 'aggregator') + GROUP BY sc.category_id + ) sub + WHERE c.id = sub.category_id + AND c.skill_count IS DISTINCT FROM sub.cnt + `); + console.log(` Updated ${result.rowCount} category counts (browse-ready only)`); + + // Also zero out categories that have no browse-ready skills + const zeroResult = await query(` + UPDATE categories c + SET skill_count = 0 + WHERE c.skill_count > 0 + AND NOT EXISTS ( + SELECT 1 FROM skill_categories sc + JOIN skills s ON sc.skill_id = s.id + WHERE sc.category_id = c.id + AND s.is_blocked = false + AND s.is_duplicate = false + AND (s.skill_type IS NULL OR s.skill_type != 'aggregator') + ) + `); + if (zeroResult.rowCount > 0) { + console.log(` Zeroed ${zeroResult.rowCount} categories with no browse-ready skills`); + } + + // Show category counts + const cats = await query(` + SELECT name, skill_count FROM categories + WHERE id NOT LIKE 'parent-%' + ORDER BY skill_count DESC + LIMIT 10 + `); + console.log(' Top 10 categories by browse-ready count:'); + for (const r of cats.rows) { + console.log(` ${n(r.skill_count).padStart(6)} ${r.name}`); + } +} + +// ─── Step 8: Summary ─── +async function step8_summary() { + header(8, 'FINAL SUMMARY'); + + const summary = await query(` + SELECT + COUNT(*)::int AS total, + COUNT(*) FILTER (WHERE is_duplicate = false)::int AS unique_skills, + COUNT(*) FILTER (WHERE is_duplicate = true)::int AS duplicates, + COUNT(*) FILTER (WHERE skill_type = 'standalone' AND is_duplicate = false)::int AS standalone, + COUNT(*) FILTER (WHERE skill_type = 'collection' AND is_duplicate = false)::int AS collection, + COUNT(*) FILTER (WHERE skill_type = 'aggregator' AND is_duplicate = false)::int AS aggregator, + COUNT(*) FILTER (WHERE skill_type = 'project-bound' AND is_duplicate = false)::int AS project_bound, + COUNT(*) FILTER (WHERE skill_type IS NULL AND is_duplicate = false)::int AS unclassified + FROM skills WHERE is_blocked = false + `); + const s = summary.rows[0]; + + console.log(` + ┌─────────────────────────────────────────────────────┐ + │ CURATION SUMMARY │ + ├─────────────────────────────────────────────────────┤ + │ Total skills: ${n(s.total).padStart(8)} │ + │ Unique skills: ${n(s.unique_skills).padStart(8)} │ + │ Duplicates: ${n(s.duplicates).padStart(8)} │ + ├─────────────────────────────────────────────────────┤ + │ Unique by type: │ + │ Standalone: ${n(s.standalone).padStart(8)} │ + │ Collection: ${n(s.collection).padStart(8)} │ + │ Aggregator: ${n(s.aggregator).padStart(8)} │ + │ Project-bound: ${n(s.project_bound).padStart(8)} │ + │ Unclassified: ${n(s.unclassified).padStart(8)} │ + └─────────────────────────────────────────────────────┘ + `); + + // Interesting standalone skills + const topStandalone = await query(` + SELECT id, name, github_stars AS stars, COALESCE(download_count,0) AS dl, + LEFT(description, 70) AS description + FROM skills + WHERE is_blocked = false AND is_duplicate = false + AND skill_type = 'standalone' + ORDER BY github_stars DESC NULLS LAST + LIMIT 20 + `); + console.log(' Top 20 standalone skills by stars:'); + for (const r of topStandalone.rows) { + console.log(` [${n(r.stars)}★ ${n(r.dl)}↓] ${r.id}`); + console.log(` ${r.description}`); + } + + // Browse-ready count (what users would see) + const browseReady = await query(` + SELECT COUNT(*)::int AS count + FROM skills + WHERE is_blocked = false + AND is_duplicate = false + AND skill_type IN ('standalone', 'collection') + `); + console.log(`\n Browse-ready skills (standalone + collection, unique): ${n(browseReady.rows[0].count)}`); +} + +// ─── Main ─── +async function main() { + const t0 = Date.now(); + await connect(); + + if (DRY_RUN) console.log('\n *** DRY RUN MODE — no changes will be made ***\n'); + + const steps = [ + step1_repoSkillCount, + step2_markAggregators, + step3_classifyRemaining, + step4_fillContentHash, + step5_markDuplicates, + step6_detectForks, + step7_categoryCounts, + step8_summary, + ]; + + for (let i = 0; i < steps.length; i++) { + if (ONLY_STEP && ONLY_STEP !== i + 1) continue; + await steps[i](); + } + + const dur = ((Date.now() - t0) / 1000).toFixed(1); + console.log(`\nCompleted in ${dur}s`); + + await client.end().catch(() => {}); +} + +main().catch(async e => { + console.error('FAILED:', e.message || e); + await client?.end().catch(() => {}); + process.exit(1); +}); diff --git a/scripts/curation/curate.sql b/scripts/curation/curate.sql new file mode 100644 index 0000000..b20f5e0 --- /dev/null +++ b/scripts/curation/curate.sql @@ -0,0 +1,214 @@ +-- ============================================================ +-- Phase 2: Data Cleanup & Classification (Pure SQL version) +-- Run inside DB container: +-- psql -U postgres -d skillhub -f /tmp/curate.sql +-- +-- Safe to run multiple times (idempotent). +-- ============================================================ + +\echo '======================================================================' +\echo ' STEP 1: COMPUTE repo_skill_count' +\echo '======================================================================' + +UPDATE skills s SET repo_skill_count = sub.cnt +FROM ( + SELECT github_owner, github_repo, COUNT(*)::int AS cnt + FROM skills WHERE is_blocked = false + GROUP BY github_owner, github_repo +) sub +WHERE s.github_owner = sub.github_owner + AND s.github_repo = sub.github_repo + AND s.is_blocked = false; + +\echo ' Step 1 done. Checking counts:' +SELECT 'repo_skill_count set' AS step, + COUNT(*) FILTER (WHERE repo_skill_count IS NOT NULL) AS done, + COUNT(*) FILTER (WHERE repo_skill_count IS NULL) AS remaining +FROM skills WHERE is_blocked = false; + + +\echo '' +\echo '======================================================================' +\echo ' STEP 2: MARK AGGREGATORS (repos with 50+ skills)' +\echo '======================================================================' + +UPDATE skills s SET skill_type = 'aggregator' +FROM ( + SELECT github_owner, github_repo + FROM skills WHERE is_blocked = false + GROUP BY github_owner, github_repo + HAVING COUNT(*) >= 50 +) agg +WHERE s.github_owner = agg.github_owner + AND s.github_repo = agg.github_repo + AND s.is_blocked = false + AND s.skill_type IS NULL; + +\echo ' Step 2 done.' + + +\echo '' +\echo '======================================================================' +\echo ' STEP 3: CLASSIFY REMAINING REPOS' +\echo '======================================================================' + +-- 3a: 10-49 skills + name contains marketplace/awesome/collection/registry → aggregator +UPDATE skills s SET skill_type = 'aggregator' +FROM ( + SELECT github_owner, github_repo + FROM skills WHERE is_blocked = false AND skill_type IS NULL + GROUP BY github_owner, github_repo + HAVING COUNT(*) >= 10 + AND (github_repo ILIKE '%marketplace%' + OR github_repo ILIKE '%awesome%' + OR github_repo ILIKE '%collection%' + OR github_repo ILIKE '%registry%') +) agg +WHERE s.github_owner = agg.github_owner + AND s.github_repo = agg.github_repo + AND s.is_blocked = false + AND s.skill_type IS NULL; + +-- 3b: 3+ skills → collection +UPDATE skills s SET skill_type = 'collection' +FROM ( + SELECT github_owner, github_repo + FROM skills WHERE is_blocked = false AND skill_type IS NULL + GROUP BY github_owner, github_repo + HAVING COUNT(*) >= 3 +) col +WHERE s.github_owner = col.github_owner + AND s.github_repo = col.github_repo + AND s.is_blocked = false + AND s.skill_type IS NULL; + +-- 3c: project-bound name patterns +UPDATE skills SET skill_type = 'project-bound' +WHERE is_blocked = false AND skill_type IS NULL + AND repo_skill_count <= 2 + AND (name ILIKE '%my-%' OR name ILIKE '%my\_%' ESCAPE '\' + OR name ILIKE '%project%' OR name ILIKE '%team%' OR name ILIKE '%internal%' + OR name ILIKE '%.mdc' OR name ILIKE '%cursorrule%' + OR name ILIKE '%config%' OR name ILIKE '%setup%'); + +-- 3d: remaining → standalone +UPDATE skills SET skill_type = 'standalone' +WHERE is_blocked = false AND skill_type IS NULL; + +\echo ' Step 3 done. Classification:' +SELECT skill_type, COUNT(*)::int AS count +FROM skills WHERE is_blocked = false +GROUP BY skill_type ORDER BY count DESC; + + +\echo '' +\echo '======================================================================' +\echo ' STEP 4: FILL content_hash WITH md5 (for consistency)' +\echo '======================================================================' + +-- Re-hash ALL with md5 for consistent dedup +UPDATE skills SET content_hash = md5(raw_content) +WHERE is_blocked = false AND raw_content IS NOT NULL; + +\echo ' Step 4 done.' +SELECT 'content_hash' AS step, + COUNT(*) FILTER (WHERE content_hash IS NOT NULL) AS has_hash, + COUNT(*) FILTER (WHERE content_hash IS NULL) AS no_hash +FROM skills WHERE is_blocked = false; + + +\echo '' +\echo '======================================================================' +\echo ' STEP 5: MARK DUPLICATES BY content_hash' +\echo '======================================================================' + +-- First reset any previous duplicate marks (for idempotency) +UPDATE skills SET is_duplicate = false, canonical_skill_id = NULL +WHERE is_blocked = false AND is_duplicate = true; + +-- Mark duplicates: keep the one with most stars (or oldest) as canonical +WITH ranked AS ( + SELECT id, content_hash, + ROW_NUMBER() OVER ( + PARTITION BY content_hash + ORDER BY github_stars DESC NULLS LAST, + created_at ASC + ) AS rn + FROM skills + WHERE is_blocked = false AND content_hash IS NOT NULL +), +canonicals AS ( + SELECT content_hash, id AS canonical_id + FROM ranked WHERE rn = 1 +) +UPDATE skills s +SET is_duplicate = true, + canonical_skill_id = c.canonical_id +FROM ranked r +JOIN canonicals c ON r.content_hash = c.content_hash +WHERE s.id = r.id + AND r.rn > 1; + +\echo ' Step 5 done.' +SELECT 'dedup' AS step, + COUNT(*) FILTER (WHERE is_duplicate = false) AS unique_skills, + COUNT(*) FILTER (WHERE is_duplicate = true) AS duplicates +FROM skills WHERE is_blocked = false; + + +\echo '' +\echo '======================================================================' +\echo ' STEP 6: DETECT FORK MARKETPLACE REPOS' +\echo '======================================================================' + +-- Re-classify fork repos (same repo name in 3+ owners with 20+ skills) as aggregator +UPDATE skills SET skill_type = 'aggregator' +WHERE is_blocked = false + AND repo_skill_count >= 20 + AND (skill_type IS NULL OR skill_type != 'aggregator') + AND github_repo IN ( + SELECT github_repo + FROM skills + WHERE is_blocked = false AND repo_skill_count >= 20 + GROUP BY github_repo + HAVING COUNT(DISTINCT github_owner) >= 3 + ); + +\echo ' Step 6 done. Fork patterns:' +SELECT github_repo, COUNT(DISTINCT github_owner)::int AS owners, + COUNT(*)::int AS total_skills +FROM skills +WHERE is_blocked = false AND repo_skill_count >= 20 +GROUP BY github_repo +HAVING COUNT(DISTINCT github_owner) >= 3 +ORDER BY owners DESC +LIMIT 15; + + +\echo '' +\echo '======================================================================' +\echo ' STEP 7: FINAL SUMMARY' +\echo '======================================================================' + +SELECT + COUNT(*)::int AS total, + COUNT(*) FILTER (WHERE is_duplicate = false)::int AS unique_skills, + COUNT(*) FILTER (WHERE is_duplicate = true)::int AS duplicates, + COUNT(*) FILTER (WHERE skill_type = 'standalone' AND is_duplicate = false)::int AS standalone, + COUNT(*) FILTER (WHERE skill_type = 'collection' AND is_duplicate = false)::int AS collection_type, + COUNT(*) FILTER (WHERE skill_type = 'aggregator' AND is_duplicate = false)::int AS aggregator, + COUNT(*) FILTER (WHERE skill_type = 'project-bound' AND is_duplicate = false)::int AS project_bound, + COUNT(*) FILTER (WHERE skill_type IN ('standalone','collection') AND is_duplicate = false)::int AS browse_ready +FROM skills WHERE is_blocked = false; + +\echo '' +\echo ' Top 20 standalone skills by stars:' +SELECT id, github_stars AS stars, COALESCE(download_count,0) AS dl, + LEFT(description, 70) AS description +FROM skills +WHERE is_blocked = false AND is_duplicate = false AND skill_type = 'standalone' +ORDER BY github_stars DESC NULLS LAST +LIMIT 20; + +\echo '' +\echo ' DONE!' diff --git a/scripts/curation/explore.mjs b/scripts/curation/explore.mjs new file mode 100644 index 0000000..be0bc70 --- /dev/null +++ b/scripts/curation/explore.mjs @@ -0,0 +1,652 @@ +#!/usr/bin/env node +/** + * Phase 1: Database Exploration — Single-Query Version + * + * Runs ALL analytics in ONE SQL query to handle unstable connections. + * + * Usage: + * PGSSLMODE=disable DATABASE_URL=postgres://... node scripts/curation/explore.mjs + */ + +import { createRequire } from 'module'; +import { writeFileSync } from 'fs'; +import { resolve, dirname } from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const projectRoot = resolve(__dirname, '../..'); + +let pg; +const tryPaths = [ + '/tmp/package.json', + process.env.APPDATA ? resolve(process.env.APPDATA, '..', 'Local', 'Temp', 'package.json') : null, + resolve(projectRoot, 'package.json'), +].filter(Boolean); +for (const p of tryPaths) { + try { pg = createRequire(p)('pg'); break; } catch {} +} +if (!pg) { console.error('pg not found. Run: cd /tmp && npm install pg'); process.exit(1); } + +const DATABASE_URL = process.env.DATABASE_URL || 'postgresql://postgres:postgres@localhost:5432/skillhub'; + +process.on('uncaughtException', e => { + if (e.code === 'ECONNRESET') return; + console.error('Fatal:', e); process.exit(1); +}); + +// ─── Mega Query ────────────────────────────────────────── +// All analytics combined into one JSON result + +const MEGA_SQL = ` +WITH active AS ( + SELECT * FROM skills WHERE is_blocked = false +), + +-- 1. Overall +overall AS ( + SELECT json_build_object( + 'total', (SELECT COUNT(*)::int FROM skills), + 'active', (SELECT COUNT(*)::int FROM active), + 'blocked', (SELECT COUNT(*) FILTER (WHERE is_blocked)::int FROM skills), + 'owners', (SELECT COUNT(DISTINCT github_owner)::int FROM active), + 'repos', (SELECT COUNT(DISTINCT github_owner||'/'||github_repo)::int FROM active), + 'skillmd', (SELECT COUNT(*) FILTER (WHERE source_format='skill.md')::int FROM active), + 'other_fmt', (SELECT COUNT(*) FILTER (WHERE source_format!='skill.md')::int FROM active), + 'downloads', (SELECT COALESCE(SUM(download_count),0)::bigint FROM active), + 'views', (SELECT COALESCE(SUM(view_count),0)::bigint FROM active), + 'rated', (SELECT COUNT(*) FILTER (WHERE rating_count>0)::int FROM active), + 'first', (SELECT MIN(created_at)::text FROM active), + 'last', (SELECT MAX(created_at)::text FROM active) + ) AS data +), + +-- 1b. Source formats +source_formats AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT COALESCE(source_format,'null') AS format, COUNT(*)::int AS count, + COUNT(*) FILTER (WHERE is_blocked=false)::int AS active + FROM skills GROUP BY source_format ORDER BY count DESC + ) t +), + +-- 2. Stars distribution +stars_dist AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT CASE + WHEN github_stars IS NULL OR github_stars=0 THEN '0' + WHEN github_stars BETWEEN 1 AND 5 THEN '1-5' + WHEN github_stars BETWEEN 6 AND 10 THEN '6-10' + WHEN github_stars BETWEEN 11 AND 50 THEN '11-50' + WHEN github_stars BETWEEN 51 AND 100 THEN '51-100' + WHEN github_stars BETWEEN 101 AND 500 THEN '101-500' + WHEN github_stars BETWEEN 501 AND 1000 THEN '501-1K' + WHEN github_stars BETWEEN 1001 AND 5000 THEN '1K-5K' + WHEN github_stars BETWEEN 5001 AND 10000 THEN '5K-10K' + WHEN github_stars BETWEEN 10001 AND 50000 THEN '10K-50K' + ELSE '50K+' + END AS stars, COUNT(*)::int AS skills, + COUNT(DISTINCT github_owner)::int AS owners, + COUNT(DISTINCT github_owner||'/'||github_repo)::int AS repos + FROM active GROUP BY 1 ORDER BY MIN(COALESCE(github_stars,0)) + ) t +), + +-- 3. Downloads +dl_dist AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT CASE + WHEN COALESCE(download_count,0)=0 THEN '0' + WHEN download_count BETWEEN 1 AND 5 THEN '1-5' + WHEN download_count BETWEEN 6 AND 50 THEN '6-50' + WHEN download_count BETWEEN 51 AND 500 THEN '51-500' + ELSE '500+' + END AS range, COUNT(*)::int AS skills + FROM active GROUP BY 1 ORDER BY MIN(COALESCE(download_count,0)) + ) t +), + +-- 3b. Views +vw_dist AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT CASE + WHEN COALESCE(view_count,0)=0 THEN '0' + WHEN view_count BETWEEN 1 AND 10 THEN '1-10' + WHEN view_count BETWEEN 11 AND 100 THEN '11-100' + WHEN view_count BETWEEN 101 AND 1000 THEN '101-1K' + ELSE '1K+' + END AS range, COUNT(*)::int AS skills + FROM active GROUP BY 1 ORDER BY MIN(COALESCE(view_count,0)) + ) t +), + +-- 3c. Top downloaded +top_dl AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT id, name, download_count AS dl, view_count AS views, + github_stars AS stars, github_owner AS owner + FROM active WHERE COALESCE(download_count,0)>0 + ORDER BY download_count DESC LIMIT 20 + ) t +), + +-- 3d. Top viewed +top_vw AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT id, name, view_count AS views, download_count AS dl, + github_stars AS stars, github_owner AS owner + FROM active WHERE COALESCE(view_count,0)>0 + ORDER BY view_count DESC LIMIT 20 + ) t +), + +-- 4. Security +security AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT COALESCE(security_status,'null') AS status, COUNT(*)::int AS count + FROM active GROUP BY security_status ORDER BY count DESC + ) t +), + +-- 5. Content stats +content_stats AS ( + SELECT json_build_object( + 'has_content', COUNT(*) FILTER (WHERE raw_content IS NOT NULL)::int, + 'no_content', COUNT(*) FILTER (WHERE raw_content IS NULL)::int, + 'avg_len', ROUND(AVG(LENGTH(raw_content)) FILTER (WHERE raw_content IS NOT NULL))::int, + 'max_len', MAX(LENGTH(raw_content))::int, + 'median_len', PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY LENGTH(raw_content)) + FILTER (WHERE raw_content IS NOT NULL)::int, + 'avg_desc', ROUND(AVG(LENGTH(description)))::int, + 'good_desc', COUNT(*) FILTER (WHERE LENGTH(description)>100)::int, + 'short_desc', COUNT(*) FILTER (WHERE LENGTH(description)<=20)::int, + 'has_ver', COUNT(*) FILTER (WHERE version IS NOT NULL)::int, + 'has_lic', COUNT(*) FILTER (WHERE license IS NOT NULL)::int, + 'has_auth', COUNT(*) FILTER (WHERE author IS NOT NULL)::int, + 'has_hp', COUNT(*) FILTER (WHERE homepage IS NOT NULL)::int + ) AS data FROM active +), + +-- 5b. Content length dist +content_len AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT CASE + WHEN raw_content IS NULL THEN 'null' + WHEN LENGTH(raw_content)=0 THEN 'empty' + WHEN LENGTH(raw_content)<200 THEN '<200' + WHEN LENGTH(raw_content)<500 THEN '200-500' + WHEN LENGTH(raw_content)<1000 THEN '500-1K' + WHEN LENGTH(raw_content)<3000 THEN '1K-3K' + WHEN LENGTH(raw_content)<5000 THEN '3K-5K' + WHEN LENGTH(raw_content)<10000 THEN '5K-10K' + WHEN LENGTH(raw_content)<50000 THEN '10K-50K' + ELSE '50K+' + END AS range, COUNT(*)::int AS skills + FROM active GROUP BY 1 ORDER BY MIN(COALESCE(LENGTH(raw_content),-1)) + ) t +), + +-- 6. Triggers +trigger_stats AS ( + SELECT json_build_object( + 'has_triggers', COUNT(*) FILTER (WHERE triggers IS NOT NULL)::int, + 'no_triggers', COUNT(*) FILTER (WHERE triggers IS NULL)::int, + 'file_pats', COUNT(*) FILTER (WHERE triggers->>'filePatterns' IS NOT NULL + AND triggers->>'filePatterns'!='[]' AND triggers->>'filePatterns'!='null')::int, + 'keywords', COUNT(*) FILTER (WHERE triggers->>'keywords' IS NOT NULL + AND triggers->>'keywords'!='[]' AND triggers->>'keywords'!='null')::int, + 'languages', COUNT(*) FILTER (WHERE triggers->>'languages' IS NOT NULL + AND triggers->>'languages'!='[]' AND triggers->>'languages'!='null')::int, + 'has_compat', COUNT(*) FILTER (WHERE compatibility IS NOT NULL)::int, + 'platforms', COUNT(*) FILTER (WHERE compatibility->>'platforms' IS NOT NULL + AND compatibility->>'platforms'!='[]' AND compatibility->>'platforms'!='null')::int + ) AS data FROM active +), + +-- 7. Freshness +freshness AS ( + SELECT json_build_object( + 'created', (SELECT json_agg(row_to_json(t)) FROM ( + SELECT CASE + WHEN created_at>NOW()-INTERVAL '7 days' THEN 'Last 7d' + WHEN created_at>NOW()-INTERVAL '30 days' THEN 'Last 30d' + WHEN created_at>NOW()-INTERVAL '90 days' THEN 'Last 90d' + WHEN created_at>NOW()-INTERVAL '180 days' THEN 'Last 180d' + ELSE 'Older' + END AS range, COUNT(*)::int AS skills + FROM active GROUP BY 1 ORDER BY MIN(NOW()-created_at) + ) t), + 'updated', (SELECT json_agg(row_to_json(t)) FROM ( + SELECT CASE + WHEN updated_at>NOW()-INTERVAL '7 days' THEN 'Last 7d' + WHEN updated_at>NOW()-INTERVAL '30 days' THEN 'Last 30d' + WHEN updated_at>NOW()-INTERVAL '90 days' THEN 'Last 90d' + WHEN updated_at>NOW()-INTERVAL '180 days' THEN 'Last 180d' + ELSE 'Older' + END AS range, COUNT(*)::int AS skills + FROM active GROUP BY 1 ORDER BY MIN(NOW()-updated_at) + ) t), + 'last_dl', (SELECT json_agg(row_to_json(t)) FROM ( + SELECT CASE + WHEN last_downloaded_at IS NULL THEN 'Never' + WHEN last_downloaded_at>NOW()-INTERVAL '30 days' THEN 'Last 30d' + WHEN last_downloaded_at>NOW()-INTERVAL '90 days' THEN 'Last 90d' + ELSE 'Older' + END AS range, COUNT(*)::int AS skills + FROM active GROUP BY 1 ORDER BY MIN(COALESCE(last_downloaded_at,'1970-01-01'::timestamp)) + ) t) + ) AS data +), + +-- 8. Top owners +top_owners_count AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT github_owner AS owner, COUNT(*)::int AS skills, + COUNT(DISTINCT github_repo)::int AS repos, + MAX(github_stars)::int AS max_stars, + COALESCE(SUM(download_count),0)::int AS dl + FROM active GROUP BY github_owner ORDER BY skills DESC LIMIT 30 + ) t +), +top_owners_stars AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT github_owner AS owner, MAX(github_stars)::int AS max_stars, + COUNT(*)::int AS skills, COUNT(DISTINCT github_repo)::int AS repos + FROM active GROUP BY github_owner ORDER BY max_stars DESC LIMIT 20 + ) t +), +owner_concentration AS ( + SELECT json_build_object( + 'top1', SUM(cnt) FILTER (WHERE rn<=1)::int, + 'top5', SUM(cnt) FILTER (WHERE rn<=5)::int, + 'top10', SUM(cnt) FILTER (WHERE rn<=10)::int, + 'top20', SUM(cnt) FILTER (WHERE rn<=20)::int, + 'top50', SUM(cnt) FILTER (WHERE rn<=50)::int, + 'total', SUM(cnt)::int, + 'owners', COUNT(*)::int + ) AS data FROM ( + SELECT github_owner, COUNT(*)::int AS cnt, + ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn + FROM active GROUP BY github_owner + ) x +), + +-- 9. Multi-skill repos +aggregators AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT github_owner||'/'||github_repo AS repo, COUNT(*)::int AS skills, + MAX(github_stars)::int AS stars + FROM active GROUP BY github_owner, github_repo HAVING COUNT(*)>=20 + ORDER BY skills DESC LIMIT 30 + ) t +), +collections AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT github_owner||'/'||github_repo AS repo, COUNT(*)::int AS skills, + MAX(github_stars)::int AS stars + FROM active GROUP BY github_owner, github_repo HAVING COUNT(*) BETWEEN 3 AND 19 + ORDER BY skills DESC LIMIT 30 + ) t +), +skills_per_repo AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + WITH rc AS (SELECT COUNT(*)::int AS cnt FROM active GROUP BY github_owner, github_repo) + SELECT CASE + WHEN cnt=1 THEN '1' WHEN cnt=2 THEN '2' + WHEN cnt BETWEEN 3 AND 5 THEN '3-5' WHEN cnt BETWEEN 6 AND 10 THEN '6-10' + WHEN cnt BETWEEN 11 AND 20 THEN '11-20' WHEN cnt BETWEEN 21 AND 50 THEN '21-50' + WHEN cnt BETWEEN 51 AND 100 THEN '51-100' ELSE '100+' + END AS per_repo, COUNT(*)::int AS repos, SUM(cnt)::int AS total_skills + FROM rc GROUP BY 1 ORDER BY MIN(cnt) + ) t +), + +-- 10. Categories +categories_dist AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT c.name AS category, c.skill_count::int AS skills + FROM categories c WHERE c.id NOT LIKE 'parent-%' ORDER BY c.skill_count DESC + ) t +), +uncategorized AS ( + SELECT COUNT(*)::int AS data FROM active s + WHERE NOT EXISTS (SELECT 1 FROM skill_categories sc WHERE sc.skill_id=s.id) +), + +-- 11. Flags +flags AS ( + SELECT json_build_object( + 'verified', COUNT(*) FILTER (WHERE is_verified)::int, + 'featured', COUNT(*) FILTER (WHERE is_featured)::int, + 'user_rated', COUNT(*) FILTER (WHERE rating_count>0)::int, + 'avg_rating', ROUND(AVG(rating) FILTER (WHERE rating IS NOT NULL),2)::numeric, + 'max_ratings', MAX(rating_count)::int + ) AS data FROM active +), + +-- 12. Cross-table +cross_stats AS ( + SELECT json_build_object( + 'users', (SELECT COUNT(*)::int FROM users), + 'ratings', (SELECT COUNT(*)::int FROM ratings), + 'installs', (SELECT COUNT(*)::int FROM installations), + 'favorites', (SELECT COUNT(*)::int FROM favorites), + 'disc_repos', (SELECT COUNT(*)::int FROM discovered_repos), + 'disc_with_skills', (SELECT COUNT(*) FILTER (WHERE has_skill_md)::int FROM discovered_repos), + 'removals', (SELECT COUNT(*)::int FROM removal_requests), + 'adds', (SELECT COUNT(*)::int FROM add_requests), + 'subs', (SELECT COUNT(*) FILTER (WHERE unsubscribed_at IS NULL)::int FROM email_subscriptions) + ) AS data +), +install_platform AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT platform, COUNT(*)::int AS count FROM installations GROUP BY platform ORDER BY count DESC + ) t +), + +-- 13. Usability signals +usability AS ( + SELECT json_build_object( + 'strong', COUNT(*) FILTER (WHERE LENGTH(description)>50 AND raw_content IS NOT NULL + AND LENGTH(raw_content)>500 AND security_status='pass')::int, + 'file_triggers', COUNT(*) FILTER (WHERE triggers IS NOT NULL AND triggers!='{}' + AND triggers->>'filePatterns' IS NOT NULL AND triggers->>'filePatterns'!='[]')::int, + 'downloaded', COUNT(*) FILTER (WHERE COALESCE(download_count,0)>0)::int, + 'high_q', COUNT(*) FILTER (WHERE github_stars>=10 AND LENGTH(description)>50 + AND raw_content IS NOT NULL AND security_status='pass')::int, + 'premium', COUNT(*) FILTER (WHERE github_stars>=100 AND COALESCE(download_count,0)>0 + AND security_status='pass')::int, + 'skillmd_q', COUNT(*) FILTER (WHERE source_format='skill.md' AND LENGTH(description)>50 + AND raw_content IS NOT NULL AND LENGTH(raw_content)>300 AND security_status='pass')::int + ) AS data FROM active +), + +-- 14. Standalone vs project-bound +repo_types AS ( + SELECT json_build_object( + 'single', COUNT(*) FILTER (WHERE cnt=1)::int, + 'two', COUNT(*) FILTER (WHERE cnt=2)::int, + 'multi', COUNT(*) FILTER (WHERE cnt>=3)::int, + 'total', COUNT(*)::int + ) AS data FROM ( + SELECT COUNT(*)::int AS cnt FROM active GROUP BY github_owner, github_repo + ) x +), +name_patterns AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT CASE + WHEN name ILIKE '%rule%' OR name ILIKE '%config%' OR name ILIKE '%setup%' THEN 'rules/config/setup' + WHEN name ILIKE '%my-%' OR name ILIKE '%my_%' THEN 'my-prefix' + WHEN name ILIKE '%cursor%' OR name ILIKE '%claude%' OR name ILIKE '%copilot%' THEN 'tool-specific' + WHEN name ILIKE '%project%' OR name ILIKE '%team%' OR name ILIKE '%internal%' THEN 'project/team' + WHEN name ILIKE '%.mdc' OR name ILIKE '%cursorrule%' THEN 'cursor-rules' + ELSE 'generic' + END AS pattern, COUNT(*)::int AS skills + FROM active GROUP BY 1 ORDER BY skills DESC + ) t +), + +-- 15. Samples +top_by_stars AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT id, name, github_stars AS stars, COALESCE(download_count,0) AS dl, + security_status AS sec, source_format AS fmt, LEFT(description,80) AS description + FROM active ORDER BY github_stars DESC LIMIT 15 + ) t +), +top_by_dl AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT id, name, download_count AS dl, view_count AS views, + github_stars AS stars, LEFT(description,80) AS description + FROM active WHERE COALESCE(download_count,0)>0 + ORDER BY download_count DESC LIMIT 15 + ) t +), + +-- 16. Discovered repos +discovered AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT discovered_via AS source, COUNT(*)::int AS total, + COUNT(*) FILTER (WHERE has_skill_md)::int AS with_skills, + COUNT(*) FILTER (WHERE last_scanned IS NOT NULL)::int AS scanned + FROM discovered_repos GROUP BY discovered_via ORDER BY total DESC + ) t +), + +-- 18. Cache +cache_stats AS ( + SELECT json_build_object( + 'cached', COUNT(*) FILTER (WHERE cached_files IS NOT NULL)::int, + 'not_cached', COUNT(*) FILTER (WHERE cached_files IS NULL)::int + ) AS data FROM active +), + +-- 19. Branches +branches AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT COALESCE(branch,'null') AS branch, COUNT(*)::int AS count + FROM active GROUP BY branch ORDER BY count DESC LIMIT 10 + ) t +), + +-- 20. Duplicates +exact_dupes AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT name, LEFT(description,60) AS desc, COUNT(*)::int AS copies, + STRING_AGG(DISTINCT github_owner,', ') AS owners + FROM active WHERE description IS NOT NULL AND LENGTH(description)>10 + GROUP BY name, description HAVING COUNT(*)>1 + ORDER BY copies DESC LIMIT 15 + ) t +), +hash_dupes AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT content_hash, COUNT(*)::int AS copies, + MIN(name) AS sample, STRING_AGG(DISTINCT github_owner,', ') AS owners + FROM active WHERE content_hash IS NOT NULL + GROUP BY content_hash HAVING COUNT(*)>2 + ORDER BY copies DESC LIMIT 15 + ) t +) + +-- Final: combine everything into one JSON +SELECT json_build_object( + 'overall', (SELECT data FROM overall), + 'sourceFormats', (SELECT data FROM source_formats), + 'starsDist', (SELECT data FROM stars_dist), + 'dlDist', (SELECT data FROM dl_dist), + 'vwDist', (SELECT data FROM vw_dist), + 'topDL', (SELECT data FROM top_dl), + 'topVW', (SELECT data FROM top_vw), + 'security', (SELECT data FROM security), + 'contentStats', (SELECT data FROM content_stats), + 'contentLen', (SELECT data FROM content_len), + 'triggerStats', (SELECT data FROM trigger_stats), + 'freshness', (SELECT data FROM freshness), + 'topOwnersCount', (SELECT data FROM top_owners_count), + 'topOwnersStars', (SELECT data FROM top_owners_stars), + 'ownerConcentration', (SELECT data FROM owner_concentration), + 'aggregators', (SELECT data FROM aggregators), + 'collections', (SELECT data FROM collections), + 'skillsPerRepo', (SELECT data FROM skills_per_repo), + 'categories', (SELECT data FROM categories_dist), + 'uncategorized', (SELECT data FROM uncategorized), + 'flags', (SELECT data FROM flags), + 'crossStats', (SELECT data FROM cross_stats), + 'installPlatform', (SELECT data FROM install_platform), + 'usability', (SELECT data FROM usability), + 'repoTypes', (SELECT data FROM repo_types), + 'namePatterns', (SELECT data FROM name_patterns), + 'topByStars', (SELECT data FROM top_by_stars), + 'topByDL', (SELECT data FROM top_by_dl), + 'discovered', (SELECT data FROM discovered), + 'cacheStats', (SELECT data FROM cache_stats), + 'branches', (SELECT data FROM branches), + 'exactDupes', (SELECT data FROM exact_dupes), + 'hashDupes', (SELECT data FROM hash_dupes) +) AS report; +`; + +// ─── Helpers ───────────────────────────────────────────── + +function header(title) { + console.log(`\n${'='.repeat(70)}\n ${title}\n${'='.repeat(70)}`); +} +function sub(title) { + console.log(`\n -- ${title} ${'─'.repeat(Math.max(0, 58 - title.length))}`); +} +function tbl(rows, max = 30) { + if (!rows || rows.length === 0) { console.log(' (no data)'); return; } + const display = rows.slice(0, max); + const keys = Object.keys(display[0]); + const w = keys.map(k => Math.max(k.length, ...display.map(r => String(r[k] ?? '').length))); + console.log(' ' + keys.map((k, i) => k.padEnd(w[i])).join(' ')); + console.log(' ' + w.map(x => '-'.repeat(x)).join(' ')); + for (const row of display) + console.log(' ' + keys.map((k, i) => String(row[k] ?? '').padEnd(w[i])).join(' ')); + if (rows.length > max) console.log(` ... and ${rows.length - max} more`); +} +const n = v => Number(v ?? 0).toLocaleString('en-US'); +const p = (a, b) => (Number(b ?? 1) === 0 ? '0.0%' : (Number(a ?? 0) / Number(b) * 100).toFixed(1) + '%'); + +// ─── Run ───────────────────────────────────────────────── + +async function run() { + const t0 = Date.now(); + console.log('Connecting and running mega-query (this may take 30-60s)...'); + + const c = new pg.Client({ connectionString: DATABASE_URL, ssl: false, + connectionTimeoutMillis: 15000, query_timeout: 120000, + keepAlive: true, keepAliveInitialDelayMillis: 1000 }); + await c.connect(); + const result = await c.query(MEGA_SQL); + await c.end().catch(() => {}); + + const r = result.rows[0].report; + const dur = ((Date.now() - t0) / 1000).toFixed(1); + console.log(`Query completed in ${dur}s\n`); + + // ── Display ── + const total = r.overall.active; + + header('1. OVERALL STATISTICS'); + const ov = r.overall; + console.log(` Total skills: ${n(ov.total)}`); + console.log(` Active (not blocked): ${n(ov.active)}`); + console.log(` Blocked: ${n(ov.blocked)}`); + console.log(` Unique owners: ${n(ov.owners)}`); + console.log(` Unique repos: ${n(ov.repos)}`); + console.log(` SKILL.md format: ${n(ov.skillmd)} (${p(ov.skillmd, ov.total)})`); + console.log(` Other formats: ${n(ov.other_fmt)} (${p(ov.other_fmt, ov.total)})`); + console.log(` Total downloads: ${n(ov.downloads)}`); + console.log(` Total views: ${n(ov.views)}`); + console.log(` Rated skills: ${n(ov.rated)}`); + console.log(` Date range: ${ov.first} -> ${ov.last}`); + sub('Source Formats'); + tbl(r.sourceFormats); + + header('2. GITHUB STARS'); + tbl(r.starsDist); + + header('3. ENGAGEMENT'); + sub('Downloads'); tbl(r.dlDist); + sub('Views'); tbl(r.vwDist); + sub('Top 20 Downloaded'); tbl(r.topDL); + sub('Top 20 Viewed'); tbl(r.topVW); + + header('4. SECURITY'); + tbl(r.security); + + header('5. CONTENT'); + const cs = r.contentStats; + console.log(` Has content: ${n(cs.has_content)} (${p(cs.has_content, total)})`); + console.log(` No content: ${n(cs.no_content)} (${p(cs.no_content, total)})`); + console.log(` Avg length: ${n(cs.avg_len)} | Median: ${n(cs.median_len)} | Max: ${n(cs.max_len)}`); + console.log(` Avg desc: ${n(cs.avg_desc)} | Good(>100): ${n(cs.good_desc)} | Short(<=20): ${n(cs.short_desc)}`); + console.log(` Version: ${n(cs.has_ver)} | License: ${n(cs.has_lic)} | Author: ${n(cs.has_auth)} | HP: ${n(cs.has_hp)}`); + sub('Content Length Dist'); + tbl(r.contentLen); + + header('6. TRIGGERS'); + const tr = r.triggerStats; + console.log(` Has triggers: ${n(tr.has_triggers)} (${p(tr.has_triggers, total)})`); + console.log(` filePatterns: ${n(tr.file_pats)} | keywords: ${n(tr.keywords)} | languages: ${n(tr.languages)}`); + console.log(` Compatibility: ${n(tr.has_compat)} | platforms: ${n(tr.platforms)}`); + + header('7. FRESHNESS'); + sub('Created'); tbl(r.freshness.created); + sub('Updated'); tbl(r.freshness.updated); + sub('Last DL'); tbl(r.freshness.last_dl); + + header('8. TOP OWNERS'); + sub('By Skill Count'); tbl(r.topOwnersCount); + sub('By Stars'); tbl(r.topOwnersStars); + sub('Concentration'); + const oc = r.ownerConcentration; + console.log(` Top 1: ${n(oc.top1)} (${p(oc.top1, oc.total)}) | Top 5: ${n(oc.top5)} (${p(oc.top5, oc.total)}) | Top 10: ${n(oc.top10)} (${p(oc.top10, oc.total)})`); + console.log(` Top 20: ${n(oc.top20)} (${p(oc.top20, oc.total)}) | Top 50: ${n(oc.top50)} (${p(oc.top50, oc.total)}) | Total owners: ${n(oc.owners)}`); + + header('9. MULTI-SKILL REPOS'); + sub('Aggregators (20+)'); tbl(r.aggregators); + sub('Collections (3-19)'); tbl(r.collections); + sub('Skills-per-Repo'); tbl(r.skillsPerRepo); + + header('10. CATEGORIES'); + tbl(r.categories); + console.log(`\n Uncategorized: ${n(r.uncategorized)}`); + + header('11. FLAGS'); + const fl = r.flags; + console.log(` Verified: ${n(fl.verified)} | Featured: ${n(fl.featured)} | Rated: ${n(fl.user_rated)} | Avg: ${fl.avg_rating ?? 'N/A'}`); + + header('12. CROSS-TABLE'); + const xt = r.crossStats; + console.log(` Users: ${n(xt.users)} | Ratings: ${n(xt.ratings)} | Installs: ${n(xt.installs)} | Favs: ${n(xt.favorites)}`); + console.log(` Discovered: ${n(xt.disc_repos)} (${n(xt.disc_with_skills)} with skills) | Subs: ${n(xt.subs)}`); + sub('Install Platform'); tbl(r.installPlatform); + + header('13. USABILITY SIGNALS'); + const us = r.usability; + console.log(` Strong standalone: ${n(us.strong)} (desc>50 + content>500 + sec=pass)`); + console.log(` File triggers: ${n(us.file_triggers)}`); + console.log(` Ever downloaded: ${n(us.downloaded)}`); + console.log(` High quality: ${n(us.high_q)} (stars>=10 + desc + sec)`); + console.log(` Premium: ${n(us.premium)} (stars>=100 + dl>0 + sec)`); + console.log(` SKILL.md quality: ${n(us.skillmd_q)}`); + + header('14. STANDALONE vs PROJECT-BOUND'); + const sm = r.repoTypes; + console.log(` Single: ${n(sm.single)} (${p(sm.single,sm.total)}) | Two: ${n(sm.two)} | Multi: ${n(sm.multi)} | Total repos: ${n(sm.total)}`); + sub('Name Patterns'); tbl(r.namePatterns); + + header('15. SAMPLES'); + sub('Top by Stars'); tbl(r.topByStars); + sub('Top by DL'); tbl(r.topByDL); + + header('16. DISCOVERED REPOS'); + tbl(r.discovered); + + header('17. CACHE'); + const cf = r.cacheStats; + console.log(` Cached: ${n(cf.cached)} (${p(cf.cached,total)}) | Not: ${n(cf.not_cached)}`); + + header('19. BRANCHES'); + tbl(r.branches); + + header('20. DUPLICATES'); + sub('Same Name+Desc'); tbl(r.exactDupes); + sub('Same Hash (3+)'); tbl(r.hashDupes); + + header(`COMPLETE (${dur}s)`); + + // Save JSON + r.generatedAt = new Date().toISOString(); + r.durationSeconds = parseFloat(dur); + const reportPath = resolve(__dirname, 'explore-report.json'); + writeFileSync(reportPath, JSON.stringify(r, null, 2), 'utf-8'); + console.log(`\n Report saved: ${reportPath}`); +} + +run().catch(async e => { + console.error('FAILED:', e.message || e); + process.exit(1); +}); diff --git a/scripts/curation/explore.sql b/scripts/curation/explore.sql new file mode 100644 index 0000000..d2293ab --- /dev/null +++ b/scripts/curation/explore.sql @@ -0,0 +1,258 @@ +-- Phase 1: Database Exploration - Run inside container: +-- psql -U postgres skillhub -t -A -f /tmp/explore.sql > /tmp/report.json +-- Or paste this SQL directly in psql + +WITH active AS (SELECT * FROM skills WHERE is_blocked = false), +overall AS ( + SELECT json_build_object( + 'total', (SELECT COUNT(*)::int FROM skills), + 'active', (SELECT COUNT(*)::int FROM active), + 'blocked', (SELECT COUNT(*) FILTER (WHERE is_blocked)::int FROM skills), + 'owners', (SELECT COUNT(DISTINCT github_owner)::int FROM active), + 'repos', (SELECT COUNT(DISTINCT github_owner||'/'||github_repo)::int FROM active), + 'skillmd', (SELECT COUNT(*) FILTER (WHERE source_format='skill.md')::int FROM active), + 'other_fmt', (SELECT COUNT(*) FILTER (WHERE source_format!='skill.md')::int FROM active), + 'downloads', (SELECT COALESCE(SUM(download_count),0)::bigint FROM active), + 'views', (SELECT COALESCE(SUM(view_count),0)::bigint FROM active), + 'rated', (SELECT COUNT(*) FILTER (WHERE rating_count>0)::int FROM active), + 'first', (SELECT MIN(created_at)::text FROM active), + 'last', (SELECT MAX(created_at)::text FROM active) + ) AS data +), +source_formats AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT COALESCE(source_format,'null') AS format, COUNT(*)::int AS count + FROM skills GROUP BY source_format ORDER BY count DESC + ) t +), +stars_dist AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT CASE + WHEN github_stars IS NULL OR github_stars=0 THEN '0' + WHEN github_stars BETWEEN 1 AND 10 THEN '1-10' + WHEN github_stars BETWEEN 11 AND 50 THEN '11-50' + WHEN github_stars BETWEEN 51 AND 100 THEN '51-100' + WHEN github_stars BETWEEN 101 AND 500 THEN '101-500' + WHEN github_stars BETWEEN 501 AND 1000 THEN '501-1K' + WHEN github_stars BETWEEN 1001 AND 5000 THEN '1K-5K' + WHEN github_stars BETWEEN 5001 AND 50000 THEN '5K-50K' + ELSE '50K+' + END AS stars, COUNT(*)::int AS skills, + COUNT(DISTINCT github_owner)::int AS owners + FROM active GROUP BY 1 ORDER BY MIN(COALESCE(github_stars,0)) + ) t +), +dl_dist AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT CASE + WHEN COALESCE(download_count,0)=0 THEN '0' + WHEN download_count BETWEEN 1 AND 10 THEN '1-10' + WHEN download_count BETWEEN 11 AND 100 THEN '11-100' + ELSE '100+' + END AS range, COUNT(*)::int AS skills + FROM active GROUP BY 1 ORDER BY MIN(COALESCE(download_count,0)) + ) t +), +top_dl AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT id, name, download_count AS dl, github_stars AS stars, github_owner AS owner + FROM active WHERE COALESCE(download_count,0)>0 ORDER BY download_count DESC LIMIT 20 + ) t +), +top_vw AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT id, name, view_count AS views, github_stars AS stars, github_owner AS owner + FROM active WHERE COALESCE(view_count,0)>0 ORDER BY view_count DESC LIMIT 20 + ) t +), +security AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT COALESCE(security_status,'null') AS status, COUNT(*)::int AS count + FROM active GROUP BY security_status ORDER BY count DESC + ) t +), +content_stats AS ( + SELECT json_build_object( + 'has_content', COUNT(*) FILTER (WHERE raw_content IS NOT NULL)::int, + 'no_content', COUNT(*) FILTER (WHERE raw_content IS NULL)::int, + 'avg_len', ROUND(AVG(LENGTH(raw_content)) FILTER (WHERE raw_content IS NOT NULL))::int, + 'max_len', MAX(LENGTH(raw_content))::int, + 'median_len', PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY LENGTH(raw_content)) FILTER (WHERE raw_content IS NOT NULL)::int, + 'avg_desc', ROUND(AVG(LENGTH(description)))::int, + 'good_desc', COUNT(*) FILTER (WHERE LENGTH(description)>100)::int, + 'short_desc', COUNT(*) FILTER (WHERE LENGTH(description)<=20)::int, + 'has_ver', COUNT(*) FILTER (WHERE version IS NOT NULL)::int, + 'has_lic', COUNT(*) FILTER (WHERE license IS NOT NULL)::int, + 'has_auth', COUNT(*) FILTER (WHERE author IS NOT NULL)::int + ) AS data FROM active +), +content_len AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT CASE + WHEN raw_content IS NULL THEN 'null' + WHEN LENGTH(raw_content)<200 THEN '<200' + WHEN LENGTH(raw_content)<1000 THEN '200-1K' + WHEN LENGTH(raw_content)<5000 THEN '1K-5K' + WHEN LENGTH(raw_content)<10000 THEN '5K-10K' + WHEN LENGTH(raw_content)<50000 THEN '10K-50K' + ELSE '50K+' + END AS range, COUNT(*)::int AS skills + FROM active GROUP BY 1 ORDER BY MIN(COALESCE(LENGTH(raw_content),-1)) + ) t +), +trigger_stats AS ( + SELECT json_build_object( + 'has_triggers', COUNT(*) FILTER (WHERE triggers IS NOT NULL)::int, + 'no_triggers', COUNT(*) FILTER (WHERE triggers IS NULL)::int, + 'file_pats', COUNT(*) FILTER (WHERE triggers->>'filePatterns' IS NOT NULL AND triggers->>'filePatterns'!='[]')::int, + 'keywords', COUNT(*) FILTER (WHERE triggers->>'keywords' IS NOT NULL AND triggers->>'keywords'!='[]')::int, + 'platforms', COUNT(*) FILTER (WHERE compatibility->>'platforms' IS NOT NULL AND compatibility->>'platforms'!='[]')::int + ) AS data FROM active +), +freshness AS ( + SELECT json_build_object( + 'created', (SELECT json_agg(row_to_json(t)) FROM ( + SELECT CASE WHEN created_at>NOW()-INTERVAL '7d' THEN '7d' WHEN created_at>NOW()-INTERVAL '30d' THEN '30d' + WHEN created_at>NOW()-INTERVAL '90d' THEN '90d' ELSE 'older' END AS range, COUNT(*)::int AS skills + FROM active GROUP BY 1 ORDER BY MIN(NOW()-created_at)) t), + 'updated', (SELECT json_agg(row_to_json(t)) FROM ( + SELECT CASE WHEN updated_at>NOW()-INTERVAL '7d' THEN '7d' WHEN updated_at>NOW()-INTERVAL '30d' THEN '30d' + WHEN updated_at>NOW()-INTERVAL '90d' THEN '90d' ELSE 'older' END AS range, COUNT(*)::int AS skills + FROM active GROUP BY 1 ORDER BY MIN(NOW()-updated_at)) t) + ) AS data +), +top_owners AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT github_owner AS owner, COUNT(*)::int AS skills, COUNT(DISTINCT github_repo)::int AS repos, + MAX(github_stars)::int AS max_stars, COALESCE(SUM(download_count),0)::int AS dl + FROM active GROUP BY github_owner ORDER BY skills DESC LIMIT 30 + ) t +), +owner_concentration AS ( + SELECT json_build_object( + 'top1', SUM(cnt) FILTER (WHERE rn<=1)::int, 'top5', SUM(cnt) FILTER (WHERE rn<=5)::int, + 'top10', SUM(cnt) FILTER (WHERE rn<=10)::int, 'top20', SUM(cnt) FILTER (WHERE rn<=20)::int, + 'total', SUM(cnt)::int, 'owners', COUNT(*)::int + ) AS data FROM ( + SELECT github_owner, COUNT(*)::int AS cnt, ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn + FROM active GROUP BY github_owner + ) x +), +aggregators AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT github_owner||'/'||github_repo AS repo, COUNT(*)::int AS skills, MAX(github_stars)::int AS stars + FROM active GROUP BY github_owner, github_repo HAVING COUNT(*)>=20 ORDER BY skills DESC LIMIT 20 + ) t +), +collections AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT github_owner||'/'||github_repo AS repo, COUNT(*)::int AS skills, MAX(github_stars)::int AS stars + FROM active GROUP BY github_owner, github_repo HAVING COUNT(*) BETWEEN 3 AND 19 ORDER BY skills DESC LIMIT 20 + ) t +), +skills_per_repo AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + WITH rc AS (SELECT COUNT(*)::int AS cnt FROM active GROUP BY github_owner, github_repo) + SELECT CASE WHEN cnt=1 THEN '1' WHEN cnt=2 THEN '2' WHEN cnt BETWEEN 3 AND 10 THEN '3-10' + WHEN cnt BETWEEN 11 AND 50 THEN '11-50' ELSE '50+' END AS per_repo, + COUNT(*)::int AS repos, SUM(cnt)::int AS total_skills + FROM rc GROUP BY 1 ORDER BY MIN(cnt) + ) t +), +categories_dist AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT c.name AS category, c.skill_count::int AS skills + FROM categories c WHERE c.id NOT LIKE 'parent-%' ORDER BY c.skill_count DESC + ) t +), +flags AS ( + SELECT json_build_object( + 'verified', COUNT(*) FILTER (WHERE is_verified)::int, + 'featured', COUNT(*) FILTER (WHERE is_featured)::int, + 'user_rated', COUNT(*) FILTER (WHERE rating_count>0)::int + ) AS data FROM active +), +cross_stats AS ( + SELECT json_build_object( + 'users', (SELECT COUNT(*)::int FROM users), + 'ratings', (SELECT COUNT(*)::int FROM ratings), + 'installs', (SELECT COUNT(*)::int FROM installations), + 'favorites', (SELECT COUNT(*)::int FROM favorites), + 'disc_repos', (SELECT COUNT(*)::int FROM discovered_repos), + 'subs', (SELECT COUNT(*) FILTER (WHERE unsubscribed_at IS NULL)::int FROM email_subscriptions) + ) AS data +), +usability AS ( + SELECT json_build_object( + 'strong', COUNT(*) FILTER (WHERE LENGTH(description)>50 AND raw_content IS NOT NULL AND LENGTH(raw_content)>500 AND security_status='pass')::int, + 'downloaded', COUNT(*) FILTER (WHERE COALESCE(download_count,0)>0)::int, + 'high_q', COUNT(*) FILTER (WHERE github_stars>=10 AND LENGTH(description)>50 AND raw_content IS NOT NULL AND security_status='pass')::int, + 'premium', COUNT(*) FILTER (WHERE github_stars>=100 AND COALESCE(download_count,0)>0 AND security_status='pass')::int, + 'skillmd_q', COUNT(*) FILTER (WHERE source_format='skill.md' AND LENGTH(description)>50 AND raw_content IS NOT NULL AND LENGTH(raw_content)>300 AND security_status='pass')::int + ) AS data FROM active +), +repo_types AS ( + SELECT json_build_object( + 'single', COUNT(*) FILTER (WHERE cnt=1)::int, 'two', COUNT(*) FILTER (WHERE cnt=2)::int, + 'multi', COUNT(*) FILTER (WHERE cnt>=3)::int, 'total', COUNT(*)::int + ) AS data FROM (SELECT COUNT(*)::int AS cnt FROM active GROUP BY github_owner, github_repo) x +), +name_patterns AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT CASE + WHEN name ILIKE '%rule%' OR name ILIKE '%config%' OR name ILIKE '%setup%' THEN 'rules/config' + WHEN name ILIKE '%cursor%' OR name ILIKE '%claude%' OR name ILIKE '%copilot%' THEN 'tool-specific' + WHEN name ILIKE '%project%' OR name ILIKE '%team%' THEN 'project/team' + ELSE 'generic' + END AS pattern, COUNT(*)::int AS skills + FROM active GROUP BY 1 ORDER BY skills DESC + ) t +), +top_by_stars AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT id, name, github_stars AS stars, COALESCE(download_count,0) AS dl, + security_status AS sec, source_format AS fmt, LEFT(description,80) AS description + FROM active ORDER BY github_stars DESC LIMIT 15 + ) t +), +discovered AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT discovered_via AS source, COUNT(*)::int AS total, + COUNT(*) FILTER (WHERE has_skill_md)::int AS with_skills + FROM discovered_repos GROUP BY discovered_via ORDER BY total DESC + ) t +), +hash_dupes AS ( + SELECT json_agg(row_to_json(t)) AS data FROM ( + SELECT content_hash, COUNT(*)::int AS copies, MIN(name) AS sample + FROM active WHERE content_hash IS NOT NULL + GROUP BY content_hash HAVING COUNT(*)>2 ORDER BY copies DESC LIMIT 15 + ) t +) +SELECT json_build_object( + 'overall', (SELECT data FROM overall), + 'sourceFormats', (SELECT data FROM source_formats), + 'starsDist', (SELECT data FROM stars_dist), + 'dlDist', (SELECT data FROM dl_dist), + 'topDL', (SELECT data FROM top_dl), + 'topVW', (SELECT data FROM top_vw), + 'security', (SELECT data FROM security), + 'contentStats', (SELECT data FROM content_stats), + 'contentLen', (SELECT data FROM content_len), + 'triggerStats', (SELECT data FROM trigger_stats), + 'freshness', (SELECT data FROM freshness), + 'topOwners', (SELECT data FROM top_owners), + 'ownerConcentration', (SELECT data FROM owner_concentration), + 'aggregators', (SELECT data FROM aggregators), + 'collections', (SELECT data FROM collections), + 'skillsPerRepo', (SELECT data FROM skills_per_repo), + 'categories', (SELECT data FROM categories_dist), + 'flags', (SELECT data FROM flags), + 'crossStats', (SELECT data FROM cross_stats), + 'usability', (SELECT data FROM usability), + 'repoTypes', (SELECT data FROM repo_types), + 'namePatterns', (SELECT data FROM name_patterns), + 'topByStars', (SELECT data FROM top_by_stars), + 'discovered', (SELECT data FROM discovered), + 'hashDupes', (SELECT data FROM hash_dupes) +) AS report; diff --git a/scripts/curation/explore.ts b/scripts/curation/explore.ts new file mode 100644 index 0000000..c62467f --- /dev/null +++ b/scripts/curation/explore.ts @@ -0,0 +1,894 @@ +#!/usr/bin/env tsx +/** + * Phase 1: Database Exploration for Curation + * + * Comprehensive analysis of all indexed skills to understand: + * - Overall statistics and distribution + * - Quality signals and content analysis + * - Owner/repo concentration + * - Freshness and activity + * - Usability signals for curation decisions + * + * Usage: + * cd services/indexer && npx tsx ../../scripts/curation/explore.ts + * + * Output: prints report to console + saves JSON to scripts/curation/explore-report.json + */ + +import { createDb, closeDb, sql } from '@skillhub/db'; +import { writeFileSync } from 'fs'; +import { resolve, dirname } from 'path'; +import { fileURLToPath } from 'url'; + +// ─── Connection ──────────────────────────────────────────────────────────────── + +const db = createDb(); + +// ─── Helpers ─────────────────────────────────────────────────────────────────── + +function header(title: string) { + const line = '='.repeat(70); + console.log(`\n${line}`); + console.log(` ${title}`); + console.log(line); +} + +function subHeader(title: string) { + console.log(`\n -- ${title} ${'─'.repeat(Math.max(0, 60 - title.length))}`); +} + +function table(rows: Record[], maxRows = 30) { + if (rows.length === 0) { + console.log(' (no data)'); + return; + } + const display = rows.slice(0, maxRows); + const keys = Object.keys(display[0]); + const widths = keys.map(k => + Math.max(k.length, ...display.map(r => String(r[k] ?? '').length)) + ); + + // Header + console.log(' ' + keys.map((k, i) => k.padEnd(widths[i])).join(' ')); + console.log(' ' + widths.map(w => '-'.repeat(w)).join(' ')); + + // Rows + for (const row of display) { + console.log(' ' + keys.map((k, i) => String(row[k] ?? '').padEnd(widths[i])).join(' ')); + } + if (rows.length > maxRows) { + console.log(` ... and ${rows.length - maxRows} more rows`); + } +} + +function num(n: unknown): string { + return Number(n ?? 0).toLocaleString('en-US'); +} + +function pct(part: unknown, total: unknown): string { + const p = Number(part ?? 0); + const t = Number(total ?? 1); + if (t === 0) return '0.0%'; + return (p / t * 100).toFixed(1) + '%'; +} + +// Helper: execute raw SQL and return rows +async function query>(strings: TemplateStringsArray, ...values: unknown[]): Promise { + const result = await db.execute(sql.raw(String.raw(strings, ...values))); + return result.rows as T[]; +} + +// ─── Report accumulator ──────────────────────────────────────────────────────── + +const report: Record = {}; + +// ─── Queries ─────────────────────────────────────────────────────────────────── + +async function runExploration() { + const startTime = Date.now(); + + // ====================================================================== + // 1. OVERALL STATISTICS + // ====================================================================== + header('1. OVERALL STATISTICS'); + + const overallRows = await db.execute(sql` + SELECT + COUNT(*)::int AS total_skills, + COUNT(*) FILTER (WHERE is_blocked = false)::int AS active_skills, + COUNT(*) FILTER (WHERE is_blocked = true)::int AS blocked_skills, + COUNT(DISTINCT github_owner)::int AS unique_owners, + COUNT(DISTINCT github_owner || '/' || github_repo)::int AS unique_repos, + COUNT(*) FILTER (WHERE source_format = 'skill.md')::int AS skill_md_count, + COUNT(*) FILTER (WHERE source_format != 'skill.md')::int AS other_format_count, + COALESCE(SUM(download_count), 0)::bigint AS total_downloads, + COALESCE(SUM(view_count), 0)::bigint AS total_views, + COUNT(*) FILTER (WHERE rating_count > 0)::int AS rated_skills, + COALESCE(SUM(rating_count), 0)::int AS total_ratings, + MIN(created_at)::text AS earliest_skill, + MAX(created_at)::text AS latest_skill + FROM skills + `); + const overall = overallRows.rows[0] as Record; + report.overall = overall; + + console.log(` Total skills: ${num(overall.total_skills)}`); + console.log(` Active (not blocked): ${num(overall.active_skills)}`); + console.log(` Blocked: ${num(overall.blocked_skills)}`); + console.log(` Unique owners: ${num(overall.unique_owners)}`); + console.log(` Unique repos: ${num(overall.unique_repos)}`); + console.log(` SKILL.md format: ${num(overall.skill_md_count)} (${pct(overall.skill_md_count, overall.total_skills)})`); + console.log(` Other formats: ${num(overall.other_format_count)} (${pct(overall.other_format_count, overall.total_skills)})`); + console.log(` Total downloads: ${num(overall.total_downloads)}`); + console.log(` Total views: ${num(overall.total_views)}`); + console.log(` Rated skills: ${num(overall.rated_skills)}`); + console.log(` Total ratings: ${num(overall.total_ratings)}`); + console.log(` Date range: ${overall.earliest_skill} -> ${overall.latest_skill}`); + + const total = Number(overall.active_skills); + + // Source format distribution + subHeader('Source Format Distribution'); + const sfRows = await db.execute(sql` + SELECT + COALESCE(source_format, 'null') AS format, + COUNT(*)::int AS count, + COUNT(*) FILTER (WHERE is_blocked = false)::int AS active + FROM skills + GROUP BY source_format + ORDER BY count DESC + `); + report.sourceFormats = sfRows.rows; + table(sfRows.rows as Record[]); + + // ====================================================================== + // 2. STARS DISTRIBUTION + // ====================================================================== + header('2. GITHUB STARS DISTRIBUTION'); + + const starsRows = await db.execute(sql` + SELECT + CASE + WHEN github_stars IS NULL OR github_stars = 0 THEN '0 stars' + WHEN github_stars BETWEEN 1 AND 5 THEN '1-5' + WHEN github_stars BETWEEN 6 AND 10 THEN '6-10' + WHEN github_stars BETWEEN 11 AND 50 THEN '11-50' + WHEN github_stars BETWEEN 51 AND 100 THEN '51-100' + WHEN github_stars BETWEEN 101 AND 500 THEN '101-500' + WHEN github_stars BETWEEN 501 AND 1000 THEN '501-1K' + WHEN github_stars BETWEEN 1001 AND 5000 THEN '1K-5K' + WHEN github_stars BETWEEN 5001 AND 10000 THEN '5K-10K' + WHEN github_stars BETWEEN 10001 AND 50000 THEN '10K-50K' + ELSE '50K+' + END AS star_range, + COUNT(*)::int AS skills, + COUNT(DISTINCT github_owner)::int AS owners, + COUNT(DISTINCT github_owner || '/' || github_repo)::int AS repos + FROM skills + WHERE is_blocked = false + GROUP BY + CASE + WHEN github_stars IS NULL OR github_stars = 0 THEN 0 + WHEN github_stars BETWEEN 1 AND 5 THEN 1 + WHEN github_stars BETWEEN 6 AND 10 THEN 2 + WHEN github_stars BETWEEN 11 AND 50 THEN 3 + WHEN github_stars BETWEEN 51 AND 100 THEN 4 + WHEN github_stars BETWEEN 101 AND 500 THEN 5 + WHEN github_stars BETWEEN 501 AND 1000 THEN 6 + WHEN github_stars BETWEEN 1001 AND 5000 THEN 7 + WHEN github_stars BETWEEN 5001 AND 10000 THEN 8 + WHEN github_stars BETWEEN 10001 AND 50000 THEN 9 + ELSE 10 + END, + CASE + WHEN github_stars IS NULL OR github_stars = 0 THEN '0 stars' + WHEN github_stars BETWEEN 1 AND 5 THEN '1-5' + WHEN github_stars BETWEEN 6 AND 10 THEN '6-10' + WHEN github_stars BETWEEN 11 AND 50 THEN '11-50' + WHEN github_stars BETWEEN 51 AND 100 THEN '51-100' + WHEN github_stars BETWEEN 101 AND 500 THEN '101-500' + WHEN github_stars BETWEEN 501 AND 1000 THEN '501-1K' + WHEN github_stars BETWEEN 1001 AND 5000 THEN '1K-5K' + WHEN github_stars BETWEEN 5001 AND 10000 THEN '5K-10K' + WHEN github_stars BETWEEN 10001 AND 50000 THEN '10K-50K' + ELSE '50K+' + END + ORDER BY MIN(COALESCE(github_stars, 0)) + `); + report.starsDist = starsRows.rows; + table(starsRows.rows as Record[]); + + // ====================================================================== + // 3. ENGAGEMENT DISTRIBUTION + // ====================================================================== + header('3. ENGAGEMENT (Downloads & Views)'); + + subHeader('Download Distribution'); + const dlRows = await db.execute(sql` + SELECT + CASE + WHEN COALESCE(download_count, 0) = 0 THEN '0' + WHEN download_count BETWEEN 1 AND 5 THEN '1-5' + WHEN download_count BETWEEN 6 AND 10 THEN '6-10' + WHEN download_count BETWEEN 11 AND 50 THEN '11-50' + WHEN download_count BETWEEN 51 AND 100 THEN '51-100' + WHEN download_count BETWEEN 101 AND 500 THEN '101-500' + ELSE '500+' + END AS range, + COUNT(*)::int AS skills + FROM skills WHERE is_blocked = false + GROUP BY 1 ORDER BY MIN(COALESCE(download_count, 0)) + `); + report.downloadDist = dlRows.rows; + table(dlRows.rows as Record[]); + + subHeader('View Distribution'); + const vwRows = await db.execute(sql` + SELECT + CASE + WHEN COALESCE(view_count, 0) = 0 THEN '0' + WHEN view_count BETWEEN 1 AND 10 THEN '1-10' + WHEN view_count BETWEEN 11 AND 50 THEN '11-50' + WHEN view_count BETWEEN 51 AND 100 THEN '51-100' + WHEN view_count BETWEEN 101 AND 500 THEN '101-500' + WHEN view_count BETWEEN 501 AND 1000 THEN '501-1K' + ELSE '1K+' + END AS range, + COUNT(*)::int AS skills + FROM skills WHERE is_blocked = false + GROUP BY 1 ORDER BY MIN(COALESCE(view_count, 0)) + `); + report.viewDist = vwRows.rows; + table(vwRows.rows as Record[]); + + subHeader('Top 20 Downloaded Skills'); + const topDL = await db.execute(sql` + SELECT id, name, download_count AS downloads, view_count AS views, + github_stars AS stars, github_owner AS owner + FROM skills + WHERE is_blocked = false AND COALESCE(download_count, 0) > 0 + ORDER BY download_count DESC LIMIT 20 + `); + report.topDownloads = topDL.rows; + table(topDL.rows as Record[]); + + subHeader('Top 20 Viewed Skills'); + const topVW = await db.execute(sql` + SELECT id, name, view_count AS views, download_count AS downloads, + github_stars AS stars, github_owner AS owner + FROM skills + WHERE is_blocked = false AND COALESCE(view_count, 0) > 0 + ORDER BY view_count DESC LIMIT 20 + `); + report.topViews = topVW.rows; + table(topVW.rows as Record[]); + + // ====================================================================== + // 4. SECURITY STATUS + // ====================================================================== + header('4. SECURITY STATUS'); + + const secRows = await db.execute(sql` + SELECT + COALESCE(security_status, 'null') AS status, + COUNT(*)::int AS count, + ROUND(COUNT(*)::numeric / NULLIF(${total}, 0) * 100, 1) AS pct + FROM skills WHERE is_blocked = false + GROUP BY security_status ORDER BY count DESC + `); + report.securityDist = secRows.rows; + table(secRows.rows as Record[]); + + // ====================================================================== + // 5. CONTENT ANALYSIS + // ====================================================================== + header('5. CONTENT ANALYSIS'); + + const csRows = await db.execute(sql` + SELECT + COUNT(*) FILTER (WHERE raw_content IS NOT NULL)::int AS has_content, + COUNT(*) FILTER (WHERE raw_content IS NULL)::int AS no_content, + ROUND(AVG(LENGTH(raw_content)) FILTER (WHERE raw_content IS NOT NULL))::int AS avg_content_len, + MAX(LENGTH(raw_content))::int AS max_content_len, + PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY LENGTH(raw_content)) + FILTER (WHERE raw_content IS NOT NULL)::int AS median_content_len, + ROUND(AVG(LENGTH(description)))::int AS avg_desc_len, + COUNT(*) FILTER (WHERE LENGTH(description) > 100)::int AS good_desc, + COUNT(*) FILTER (WHERE LENGTH(description) <= 20)::int AS short_desc, + COUNT(*) FILTER (WHERE version IS NOT NULL)::int AS has_version, + COUNT(*) FILTER (WHERE license IS NOT NULL)::int AS has_license, + COUNT(*) FILTER (WHERE author IS NOT NULL)::int AS has_author, + COUNT(*) FILTER (WHERE homepage IS NOT NULL)::int AS has_homepage + FROM skills WHERE is_blocked = false + `); + const cs = csRows.rows[0] as Record; + report.contentStats = cs; + + console.log(` Has raw_content: ${num(cs.has_content)} (${pct(cs.has_content, total)})`); + console.log(` No raw_content: ${num(cs.no_content)} (${pct(cs.no_content, total)})`); + console.log(` Avg content length: ${num(cs.avg_content_len)} chars`); + console.log(` Median content len: ${num(cs.median_content_len)} chars`); + console.log(` Max content length: ${num(cs.max_content_len)} chars`); + console.log(` Avg desc length: ${num(cs.avg_desc_len)} chars`); + console.log(` Good desc (>100ch): ${num(cs.good_desc)} (${pct(cs.good_desc, total)})`); + console.log(` Short desc (<=20): ${num(cs.short_desc)} (${pct(cs.short_desc, total)})`); + console.log(` Has version: ${num(cs.has_version)} (${pct(cs.has_version, total)})`); + console.log(` Has license: ${num(cs.has_license)} (${pct(cs.has_license, total)})`); + console.log(` Has author: ${num(cs.has_author)} (${pct(cs.has_author, total)})`); + console.log(` Has homepage: ${num(cs.has_homepage)} (${pct(cs.has_homepage, total)})`); + + subHeader('Content Length Distribution'); + const clRows = await db.execute(sql` + SELECT + CASE + WHEN raw_content IS NULL THEN 'null' + WHEN LENGTH(raw_content) = 0 THEN 'empty' + WHEN LENGTH(raw_content) < 200 THEN '<200' + WHEN LENGTH(raw_content) < 500 THEN '200-500' + WHEN LENGTH(raw_content) < 1000 THEN '500-1K' + WHEN LENGTH(raw_content) < 3000 THEN '1K-3K' + WHEN LENGTH(raw_content) < 5000 THEN '3K-5K' + WHEN LENGTH(raw_content) < 10000 THEN '5K-10K' + WHEN LENGTH(raw_content) < 50000 THEN '10K-50K' + ELSE '50K+' + END AS range, + COUNT(*)::int AS skills + FROM skills WHERE is_blocked = false + GROUP BY 1 ORDER BY MIN(COALESCE(LENGTH(raw_content), -1)) + `); + report.contentLenDist = clRows.rows; + table(clRows.rows as Record[]); + + // ====================================================================== + // 6. TRIGGERS & COMPATIBILITY + // ====================================================================== + header('6. TRIGGERS & COMPATIBILITY'); + + const trRows = await db.execute(sql` + SELECT + COUNT(*) FILTER (WHERE triggers IS NOT NULL)::int AS has_triggers, + COUNT(*) FILTER (WHERE triggers IS NULL)::int AS no_triggers, + COUNT(*) FILTER (WHERE triggers->>'filePatterns' IS NOT NULL + AND triggers->>'filePatterns' != '[]' + AND triggers->>'filePatterns' != 'null')::int AS has_file_patterns, + COUNT(*) FILTER (WHERE triggers->>'keywords' IS NOT NULL + AND triggers->>'keywords' != '[]' + AND triggers->>'keywords' != 'null')::int AS has_keywords, + COUNT(*) FILTER (WHERE triggers->>'languages' IS NOT NULL + AND triggers->>'languages' != '[]' + AND triggers->>'languages' != 'null')::int AS has_languages, + COUNT(*) FILTER (WHERE compatibility IS NOT NULL)::int AS has_compat, + COUNT(*) FILTER (WHERE compatibility->>'platforms' IS NOT NULL + AND compatibility->>'platforms' != '[]' + AND compatibility->>'platforms' != 'null')::int AS has_platforms, + COUNT(*) FILTER (WHERE compatibility->>'requires' IS NOT NULL + AND compatibility->>'requires' != '[]' + AND compatibility->>'requires' != 'null')::int AS has_requires + FROM skills WHERE is_blocked = false + `); + const tr = trRows.rows[0] as Record; + report.triggerStats = tr; + + console.log(` Has triggers: ${num(tr.has_triggers)} (${pct(tr.has_triggers, total)})`); + console.log(` No triggers: ${num(tr.no_triggers)} (${pct(tr.no_triggers, total)})`); + console.log(` - filePatterns: ${num(tr.has_file_patterns)}`); + console.log(` - keywords: ${num(tr.has_keywords)}`); + console.log(` - languages: ${num(tr.has_languages)}`); + console.log(` Has compatibility: ${num(tr.has_compat)}`); + console.log(` - platforms: ${num(tr.has_platforms)}`); + console.log(` - requires: ${num(tr.has_requires)}`); + + // ====================================================================== + // 7. FRESHNESS + // ====================================================================== + header('7. FRESHNESS & ACTIVITY'); + + subHeader('Created At Distribution'); + const crRows = await db.execute(sql` + SELECT + CASE + WHEN created_at > NOW() - INTERVAL '7 days' THEN 'Last 7d' + WHEN created_at > NOW() - INTERVAL '30 days' THEN 'Last 30d' + WHEN created_at > NOW() - INTERVAL '90 days' THEN 'Last 90d' + WHEN created_at > NOW() - INTERVAL '180 days' THEN 'Last 180d' + WHEN created_at > NOW() - INTERVAL '365 days' THEN 'Last 365d' + ELSE 'Older' + END AS range, + COUNT(*)::int AS skills + FROM skills WHERE is_blocked = false + GROUP BY 1 ORDER BY MIN(NOW() - created_at) + `); + report.createdDist = crRows.rows; + table(crRows.rows as Record[]); + + subHeader('Updated At Distribution'); + const upRows = await db.execute(sql` + SELECT + CASE + WHEN updated_at > NOW() - INTERVAL '7 days' THEN 'Last 7d' + WHEN updated_at > NOW() - INTERVAL '30 days' THEN 'Last 30d' + WHEN updated_at > NOW() - INTERVAL '90 days' THEN 'Last 90d' + WHEN updated_at > NOW() - INTERVAL '180 days' THEN 'Last 180d' + WHEN updated_at > NOW() - INTERVAL '365 days' THEN 'Last 365d' + ELSE 'Older' + END AS range, + COUNT(*)::int AS skills + FROM skills WHERE is_blocked = false + GROUP BY 1 ORDER BY MIN(NOW() - updated_at) + `); + report.updatedDist = upRows.rows; + table(upRows.rows as Record[]); + + subHeader('Last Downloaded Distribution'); + const ldRows = await db.execute(sql` + SELECT + CASE + WHEN last_downloaded_at IS NULL THEN 'Never' + WHEN last_downloaded_at > NOW() - INTERVAL '7 days' THEN 'Last 7d' + WHEN last_downloaded_at > NOW() - INTERVAL '30 days' THEN 'Last 30d' + WHEN last_downloaded_at > NOW() - INTERVAL '90 days' THEN 'Last 90d' + ELSE 'Older' + END AS range, + COUNT(*)::int AS skills + FROM skills WHERE is_blocked = false + GROUP BY 1 ORDER BY MIN(COALESCE(last_downloaded_at, '1970-01-01'::timestamp)) + `); + report.lastDownDist = ldRows.rows; + table(ldRows.rows as Record[]); + + // ====================================================================== + // 8. TOP OWNERS + // ====================================================================== + header('8. TOP OWNERS'); + + subHeader('Top 30 Owners by Skill Count'); + const owRows = await db.execute(sql` + SELECT + github_owner AS owner, + COUNT(*)::int AS skills, + COUNT(DISTINCT github_repo)::int AS repos, + MAX(github_stars)::int AS max_stars, + COALESCE(SUM(download_count), 0)::int AS downloads, + COALESCE(SUM(view_count), 0)::int AS views + FROM skills WHERE is_blocked = false + GROUP BY github_owner ORDER BY skills DESC LIMIT 30 + `); + report.topOwnersByCount = owRows.rows; + table(owRows.rows as Record[]); + + subHeader('Top 20 Owners by Stars'); + const osRows = await db.execute(sql` + SELECT + github_owner AS owner, + MAX(github_stars)::int AS max_stars, + COUNT(*)::int AS skills, + COUNT(DISTINCT github_repo)::int AS repos, + COALESCE(SUM(download_count), 0)::int AS downloads + FROM skills WHERE is_blocked = false + GROUP BY github_owner ORDER BY max_stars DESC LIMIT 20 + `); + report.topOwnersByStars = osRows.rows; + table(osRows.rows as Record[]); + + subHeader('Owner Concentration'); + const ocRows = await db.execute(sql` + WITH ranked AS ( + SELECT github_owner, COUNT(*)::int AS cnt, + ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn + FROM skills WHERE is_blocked = false + GROUP BY github_owner + ) + SELECT + SUM(cnt) FILTER (WHERE rn <= 1)::int AS top_1, + SUM(cnt) FILTER (WHERE rn <= 5)::int AS top_5, + SUM(cnt) FILTER (WHERE rn <= 10)::int AS top_10, + SUM(cnt) FILTER (WHERE rn <= 20)::int AS top_20, + SUM(cnt) FILTER (WHERE rn <= 50)::int AS top_50, + SUM(cnt)::int AS total, + COUNT(*)::int AS owners + FROM ranked + `); + const oc = ocRows.rows[0] as Record; + report.ownerConcentration = oc; + + console.log(` Top 1 owner: ${num(oc.top_1)} skills (${pct(oc.top_1, oc.total)})`); + console.log(` Top 5 owners: ${num(oc.top_5)} skills (${pct(oc.top_5, oc.total)})`); + console.log(` Top 10 owners: ${num(oc.top_10)} skills (${pct(oc.top_10, oc.total)})`); + console.log(` Top 20 owners: ${num(oc.top_20)} skills (${pct(oc.top_20, oc.total)})`); + console.log(` Top 50 owners: ${num(oc.top_50)} skills (${pct(oc.top_50, oc.total)})`); + console.log(` Total owners: ${num(oc.owners)}`); + + // ====================================================================== + // 9. MULTI-SKILL REPOS + // ====================================================================== + header('9. MULTI-SKILL REPOS'); + + subHeader('Repos with 20+ Skills (Aggregator Candidates)'); + const aggRows = await db.execute(sql` + SELECT + github_owner || '/' || github_repo AS repo, + COUNT(*)::int AS skills, + MAX(github_stars)::int AS stars, + COALESCE(SUM(download_count), 0)::int AS downloads + FROM skills WHERE is_blocked = false + GROUP BY github_owner, github_repo + HAVING COUNT(*) >= 20 + ORDER BY skills DESC LIMIT 30 + `); + report.aggregatorCandidates = aggRows.rows; + table(aggRows.rows as Record[]); + + subHeader('Repos with 3-19 Skills (Collection Candidates)'); + const colRows = await db.execute(sql` + SELECT + github_owner || '/' || github_repo AS repo, + COUNT(*)::int AS skills, + MAX(github_stars)::int AS stars, + COALESCE(SUM(download_count), 0)::int AS downloads + FROM skills WHERE is_blocked = false + GROUP BY github_owner, github_repo + HAVING COUNT(*) BETWEEN 3 AND 19 + ORDER BY skills DESC LIMIT 30 + `); + report.collectionCandidates = colRows.rows; + table(colRows.rows as Record[]); + + subHeader('Skills-per-Repo Distribution'); + const sprRows = await db.execute(sql` + WITH rc AS ( + SELECT COUNT(*)::int AS cnt + FROM skills WHERE is_blocked = false + GROUP BY github_owner, github_repo + ) + SELECT + CASE + WHEN cnt = 1 THEN '1 skill' + WHEN cnt = 2 THEN '2 skills' + WHEN cnt BETWEEN 3 AND 5 THEN '3-5' + WHEN cnt BETWEEN 6 AND 10 THEN '6-10' + WHEN cnt BETWEEN 11 AND 20 THEN '11-20' + WHEN cnt BETWEEN 21 AND 50 THEN '21-50' + WHEN cnt BETWEEN 51 AND 100 THEN '51-100' + ELSE '100+' + END AS per_repo, + COUNT(*)::int AS repos, + SUM(cnt)::int AS total_skills + FROM rc + GROUP BY 1 ORDER BY MIN(cnt) + `); + report.skillsPerRepo = sprRows.rows; + table(sprRows.rows as Record[]); + + // ====================================================================== + // 10. CATEGORY DISTRIBUTION + // ====================================================================== + header('10. CATEGORY DISTRIBUTION'); + + const catRows = await db.execute(sql` + SELECT c.name AS category, c.skill_count::int AS skills, c.id + FROM categories c + WHERE c.id NOT LIKE 'parent-%' + ORDER BY c.skill_count DESC + `); + report.categoryDist = catRows.rows; + table(catRows.rows as Record[]); + + const ncRows = await db.execute(sql` + SELECT COUNT(*)::int AS count + FROM skills s + WHERE s.is_blocked = false + AND NOT EXISTS (SELECT 1 FROM skill_categories sc WHERE sc.skill_id = s.id) + `); + report.uncategorizedCount = (ncRows.rows[0] as Record)?.count; + console.log(`\n Skills with no category: ${num(report.uncategorizedCount)}`); + + // ====================================================================== + // 11. FLAGS & RATINGS + // ====================================================================== + header('11. FLAGS & RATINGS'); + + const flRows = await db.execute(sql` + SELECT + COUNT(*) FILTER (WHERE is_verified = true)::int AS verified, + COUNT(*) FILTER (WHERE is_featured = true)::int AS featured, + COUNT(*) FILTER (WHERE rating IS NOT NULL)::int AS has_rating, + COUNT(*) FILTER (WHERE rating_count > 0)::int AS has_user_ratings, + ROUND(AVG(rating) FILTER (WHERE rating IS NOT NULL), 2)::numeric AS avg_rating, + MAX(rating_count)::int AS max_rating_count + FROM skills WHERE is_blocked = false + `); + const fl = flRows.rows[0] as Record; + report.flags = fl; + + console.log(` Verified: ${num(fl.verified)}`); + console.log(` Featured: ${num(fl.featured)}`); + console.log(` Has rating: ${num(fl.has_rating)}`); + console.log(` Has user ratings: ${num(fl.has_user_ratings)}`); + console.log(` Average rating: ${fl.avg_rating ?? 'N/A'}`); + console.log(` Max rating count: ${num(fl.max_rating_count)}`); + + // ====================================================================== + // 12. CROSS-TABLE STATS + // ====================================================================== + header('12. CROSS-TABLE STATS'); + + const xtRows = await db.execute(sql` + SELECT + (SELECT COUNT(*)::int FROM users) AS users, + (SELECT COUNT(*)::int FROM ratings) AS ratings, + (SELECT COUNT(*)::int FROM installations) AS installations, + (SELECT COUNT(*)::int FROM favorites) AS favorites, + (SELECT COUNT(*)::int FROM discovered_repos) AS discovered_repos, + (SELECT COUNT(*) FILTER (WHERE has_skill_md = true)::int FROM discovered_repos) AS repos_with_skills, + (SELECT COUNT(*)::int FROM removal_requests) AS removal_requests, + (SELECT COUNT(*)::int FROM add_requests) AS add_requests, + (SELECT COUNT(*) FILTER (WHERE unsubscribed_at IS NULL)::int FROM email_subscriptions) AS subscribers + `); + const xt = xtRows.rows[0] as Record; + report.crossStats = xt; + + console.log(` Users: ${num(xt.users)}`); + console.log(` Ratings: ${num(xt.ratings)}`); + console.log(` Installations: ${num(xt.installations)}`); + console.log(` Favorites: ${num(xt.favorites)}`); + console.log(` Discovered repos: ${num(xt.discovered_repos)}`); + console.log(` - with skills: ${num(xt.repos_with_skills)}`); + console.log(` Removal requests: ${num(xt.removal_requests)}`); + console.log(` Add requests: ${num(xt.add_requests)}`); + console.log(` Subscribers: ${num(xt.subscribers)}`); + + subHeader('Installations by Platform'); + const ipRows = await db.execute(sql` + SELECT platform, COUNT(*)::int AS count + FROM installations GROUP BY platform ORDER BY count DESC + `); + report.installByPlatform = ipRows.rows; + table(ipRows.rows as Record[]); + + // ====================================================================== + // 13. USABILITY SIGNALS + // ====================================================================== + header('13. USABILITY SIGNALS FOR CURATION'); + + const usRows = await db.execute(sql` + SELECT + COUNT(*) FILTER ( + WHERE LENGTH(description) > 50 + AND raw_content IS NOT NULL AND LENGTH(raw_content) > 500 + AND security_status = 'pass' + )::int AS strong_standalone, + + COUNT(*) FILTER ( + WHERE triggers IS NOT NULL AND triggers != '{}'::jsonb + AND (triggers->>'filePatterns' IS NOT NULL AND triggers->>'filePatterns' != '[]') + )::int AS has_file_triggers, + + COUNT(*) FILTER (WHERE COALESCE(download_count, 0) > 0)::int AS ever_downloaded, + + COUNT(*) FILTER ( + WHERE github_stars >= 10 + AND LENGTH(description) > 50 + AND raw_content IS NOT NULL + AND security_status = 'pass' + )::int AS high_quality, + + COUNT(*) FILTER ( + WHERE github_stars >= 100 + AND COALESCE(download_count, 0) > 0 + AND security_status = 'pass' + )::int AS premium, + + COUNT(*) FILTER ( + WHERE source_format = 'skill.md' + AND LENGTH(description) > 50 + AND raw_content IS NOT NULL AND LENGTH(raw_content) > 300 + AND security_status = 'pass' + )::int AS skillmd_quality + FROM skills WHERE is_blocked = false + `); + const us = usRows.rows[0] as Record; + report.usability = us; + + console.log(` Strong standalone candidates: ${num(us.strong_standalone)}`); + console.log(` (desc>50 + content>500 + security=pass)`); + console.log(` With file triggers: ${num(us.has_file_triggers)}`); + console.log(` Ever downloaded: ${num(us.ever_downloaded)}`); + console.log(` High quality (stars>10+desc+sec): ${num(us.high_quality)}`); + console.log(` Premium (stars>100+dl>0+sec): ${num(us.premium)}`); + console.log(` SKILL.md with quality: ${num(us.skillmd_quality)}`); + + // ====================================================================== + // 14. STANDALONE vs PROJECT-BOUND + // ====================================================================== + header('14. STANDALONE vs PROJECT-BOUND HEURISTIC'); + + subHeader('Single vs Multi-Skill Repos'); + const smRows = await db.execute(sql` + WITH rc AS ( + SELECT github_owner, github_repo, COUNT(*)::int AS cnt + FROM skills WHERE is_blocked = false + GROUP BY github_owner, github_repo + ) + SELECT + COUNT(*) FILTER (WHERE cnt = 1)::int AS single, + COUNT(*) FILTER (WHERE cnt = 2)::int AS two, + COUNT(*) FILTER (WHERE cnt >= 3)::int AS multi, + COUNT(*)::int AS total + FROM rc + `); + const sm = smRows.rows[0] as Record; + report.repoTypes = sm; + + console.log(` Single-skill repos: ${num(sm.single)} (${pct(sm.single, sm.total)})`); + console.log(` Two-skill repos: ${num(sm.two)} (${pct(sm.two, sm.total)})`); + console.log(` Multi-skill repos: ${num(sm.multi)} (${pct(sm.multi, sm.total)})`); + + subHeader('Name Pattern Signals'); + const npRows = await db.execute(sql` + SELECT + CASE + WHEN name ILIKE '%rule%' OR name ILIKE '%config%' OR name ILIKE '%setup%' THEN 'rules/config/setup' + WHEN name ILIKE '%my-%' OR name ILIKE '%my_%' THEN 'starts with my-' + WHEN name ILIKE '%cursor%' OR name ILIKE '%claude%' OR name ILIKE '%copilot%' THEN 'tool-specific' + WHEN name ILIKE '%project%' OR name ILIKE '%team%' OR name ILIKE '%internal%' THEN 'project/team' + WHEN name ILIKE '%.mdc' OR name ILIKE '%cursorrule%' THEN 'cursor rules file' + ELSE 'generic/other' + END AS pattern, + COUNT(*)::int AS skills + FROM skills WHERE is_blocked = false + GROUP BY 1 ORDER BY skills DESC + `); + report.namePatterns = npRows.rows; + table(npRows.rows as Record[]); + + // ====================================================================== + // 15. SAMPLE SKILLS + // ====================================================================== + header('15. SAMPLE SKILLS'); + + subHeader('Top 15 by Stars'); + const tsRows = await db.execute(sql` + SELECT id, name, github_stars AS stars, COALESCE(download_count,0) AS dl, + security_status AS sec, source_format AS fmt, + LEFT(description, 80) AS description + FROM skills WHERE is_blocked = false + ORDER BY github_stars DESC LIMIT 15 + `); + report.topByStars = tsRows.rows; + table(tsRows.rows as Record[]); + + subHeader('Top 15 by Downloads (actually used)'); + const tdRows = await db.execute(sql` + SELECT id, name, download_count AS dl, view_count AS views, + github_stars AS stars, security_status AS sec, + LEFT(description, 80) AS description + FROM skills WHERE is_blocked = false AND COALESCE(download_count,0) > 0 + ORDER BY download_count DESC LIMIT 15 + `); + report.topByDL = tdRows.rows; + table(tdRows.rows as Record[]); + + subHeader('SKILL.md + Stars>50 + Downloads>0'); + const qsRows = await db.execute(sql` + SELECT id, name, github_stars AS stars, download_count AS dl, + security_status AS sec, LEFT(description, 80) AS description + FROM skills + WHERE is_blocked = false AND source_format = 'skill.md' + AND github_stars >= 50 AND COALESCE(download_count, 0) > 0 + ORDER BY github_stars DESC LIMIT 20 + `); + report.qualitySkillMd = qsRows.rows; + table(qsRows.rows as Record[]); + + // ====================================================================== + // 16. DISCOVERED REPOS + // ====================================================================== + header('16. DISCOVERED REPOS'); + + const drRows = await db.execute(sql` + SELECT + discovered_via AS source, + COUNT(*)::int AS total, + COUNT(*) FILTER (WHERE has_skill_md = true)::int AS with_skills, + COUNT(*) FILTER (WHERE last_scanned IS NOT NULL)::int AS scanned, + COUNT(*) FILTER (WHERE is_archived = true)::int AS archived + FROM discovered_repos + GROUP BY discovered_via ORDER BY total DESC + `); + report.discoveredStats = drRows.rows; + table(drRows.rows as Record[]); + + // ====================================================================== + // 17. CACHED FILES + // ====================================================================== + header('17. CACHED FILES'); + + const cfRows = await db.execute(sql` + SELECT + COUNT(*) FILTER (WHERE cached_files IS NOT NULL)::int AS has_cache, + COUNT(*) FILTER (WHERE cached_files IS NULL)::int AS no_cache + FROM skills WHERE is_blocked = false + `); + const cf = cfRows.rows[0] as Record; + report.cacheStats = cf; + + console.log(` Has cached files: ${num(cf.has_cache)} (${pct(cf.has_cache, total)})`); + console.log(` No cached files: ${num(cf.no_cache)} (${pct(cf.no_cache, total)})`); + + // ====================================================================== + // 19. BRANCH DISTRIBUTION + // ====================================================================== + header('19. BRANCH DISTRIBUTION'); + + const brRows = await db.execute(sql` + SELECT COALESCE(branch, 'null') AS branch, COUNT(*)::int AS count + FROM skills WHERE is_blocked = false + GROUP BY branch ORDER BY count DESC LIMIT 10 + `); + report.branchDist = brRows.rows; + table(brRows.rows as Record[]); + + // ====================================================================== + // 20. DUPLICATES + // ====================================================================== + header('20. POTENTIAL DUPLICATES'); + + subHeader('Same Name + Description'); + const ddRows = await db.execute(sql` + SELECT name, LEFT(description, 60) AS desc, COUNT(*)::int AS occurrences, + STRING_AGG(DISTINCT github_owner, ', ') AS owners + FROM skills + WHERE is_blocked = false AND description IS NOT NULL AND LENGTH(description) > 10 + GROUP BY name, description HAVING COUNT(*) > 1 + ORDER BY occurrences DESC LIMIT 15 + `); + report.exactDupes = ddRows.rows; + table(ddRows.rows as Record[]); + + subHeader('Same Content Hash (identical content, 3+ copies)'); + const chRows = await db.execute(sql` + SELECT content_hash, COUNT(*)::int AS copies, + MIN(name) AS sample_name, + STRING_AGG(DISTINCT github_owner, ', ') AS owners + FROM skills + WHERE is_blocked = false AND content_hash IS NOT NULL + GROUP BY content_hash HAVING COUNT(*) > 2 + ORDER BY copies DESC LIMIT 15 + `); + report.hashDupes = chRows.rows; + table(chRows.rows as Record[]); + + // ====================================================================== + // DONE + // ====================================================================== + const duration = ((Date.now() - startTime) / 1000).toFixed(1); + header(`EXPLORATION COMPLETE (${duration}s)`); + + console.log(` + KEY QUESTIONS TO ANSWER: + 1. What % of skills are SKILL.md vs other formats? + 2. What % have been downloaded at least once? (real usage) + 3. How concentrated is ownership? (top 10 owners = ?%) + 4. How many multi-skill repos? (collection/aggregator candidates) + 5. How many pass security + have good content? (curation-ready) + 6. How many duplicates exist? + `); + + // Save JSON report + report.generatedAt = new Date().toISOString(); + report.durationSeconds = parseFloat(duration); + + const scriptDir = dirname(fileURLToPath(import.meta.url)); + const reportPath = resolve(scriptDir, 'explore-report.json'); + writeFileSync(reportPath, JSON.stringify(report, null, 2), 'utf-8'); + console.log(` Full report saved to: ${reportPath}`); + + await closeDb(); +} + +// ─── Run ─────────────────────────────────────────────────────────────────────── +runExploration().catch(async (err) => { + console.error('Exploration failed:', err); + await closeDb(); + process.exit(1); +}); diff --git a/scripts/init-db.sql b/scripts/init-db.sql index 0468206..c6c233d 100644 --- a/scripts/init-db.sql +++ b/scripts/init-db.sql @@ -348,6 +348,7 @@ BEGIN -- Excluded: cached_files (download cache, not content change) -- Excluded: indexed_at, last_scanned (indexer bookkeeping) -- Excluded: github_stars, github_forks (popularity metrics, not content) + -- Excluded: quality_score, quality_details, skill_type, is_duplicate, canonical_skill_id, repo_skill_count (curation metadata) IF ROW(NEW.name, NEW.description, NEW.github_owner, NEW.github_repo, NEW.skill_path, NEW.branch, NEW.commit_sha, NEW.source_format, NEW.version, NEW.license, NEW.author, NEW.homepage, NEW.compatibility, NEW.triggers, @@ -461,6 +462,19 @@ CREATE INDEX IF NOT EXISTS idx_skills_source_format ON skills(source_format); ALTER TABLE skills ADD COLUMN IF NOT EXISTS last_downloaded_at TIMESTAMP WITH TIME ZONE; CREATE INDEX IF NOT EXISTS idx_skills_last_downloaded ON skills(last_downloaded_at DESC NULLS LAST); +-- Curation columns (Phase 2 - February 2026) +ALTER TABLE skills ADD COLUMN IF NOT EXISTS quality_score INTEGER; +ALTER TABLE skills ADD COLUMN IF NOT EXISTS quality_details JSONB; +ALTER TABLE skills ADD COLUMN IF NOT EXISTS skill_type TEXT; -- 'standalone', 'project-bound', 'collection', 'aggregator' +ALTER TABLE skills ADD COLUMN IF NOT EXISTS is_duplicate BOOLEAN DEFAULT FALSE; +ALTER TABLE skills ADD COLUMN IF NOT EXISTS canonical_skill_id TEXT; -- points to original if duplicate +ALTER TABLE skills ADD COLUMN IF NOT EXISTS repo_skill_count INTEGER; -- cached count of skills in repo + +CREATE INDEX IF NOT EXISTS idx_skills_quality ON skills(quality_score DESC NULLS LAST); +CREATE INDEX IF NOT EXISTS idx_skills_type ON skills(skill_type); +CREATE INDEX IF NOT EXISTS idx_skills_duplicate ON skills(is_duplicate); +CREATE INDEX IF NOT EXISTS idx_skills_content_hash ON skills(content_hash); + -- Grant permissions GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO postgres; GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO postgres; diff --git a/services/indexer/src/skill-indexer.ts b/services/indexer/src/skill-indexer.ts index 13e8ae5..2d9ae5a 100644 --- a/services/indexer/src/skill-indexer.ts +++ b/services/indexer/src/skill-indexer.ts @@ -90,6 +90,7 @@ export async function indexSkill( githubStars: content.repoMeta.stars, githubForks: content.repoMeta.forks, securityScore: analysis.security.score, + securityStatus: analysis.security.status, contentHash: analysis.meta.contentHash, rawContent: content.skillMd, indexedAt: new Date(),