Add comprehensive data curation system to clean up the 197K skill dataset and show only quality browse-ready skills to users. Phase 1 — Database exploration: - Explore scripts (explore.ts, explore.mjs, explore.sql) for analysis - Discovered: 69% duplicates, 77% aggregator/fork noise Phase 2 — Data cleanup and classification: - Schema: 6 new curation columns + 4 indexes - curate.mjs: 8-step pipeline (classify, dedup, fork detection, etc.) - Result: 197K → 60K unique → 16K browse-ready skills - Bug fix: securityStatus was computed but never stored during crawl Phase 3 — UI browse-ready filters: - browseReadyFilter applied to 17+ query functions - Homepage stats show accurate browse-ready counts - Stats API filtered (previously had no WHERE clause) - Category counts recalculated (e.g. 45K → 3.1K) - Featured skills exclude duplicates and aggregators Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
84 lines
2.7 KiB
TypeScript
84 lines
2.7 KiB
TypeScript
import { type NextRequest, NextResponse } from 'next/server';
|
|
import { createDb, skills, categories, sql } from '@skillhub/db';
|
|
import { getCached, setCache, cacheKeys, cacheTTL } from '@/lib/cache';
|
|
import { withRateLimit, createRateLimitResponse, createRateLimitHeaders } from '@/lib/rate-limit';
|
|
|
|
const db = createDb();
|
|
|
|
interface StatsData {
|
|
totalSkills: number;
|
|
totalDownloads: number;
|
|
totalCategories: number;
|
|
totalContributors: number;
|
|
platforms: number;
|
|
}
|
|
|
|
export async function GET(request: NextRequest) {
|
|
// Rate limiting
|
|
const rateLimitResult = await withRateLimit(request, 'anonymous');
|
|
if (!rateLimitResult.allowed) {
|
|
return createRateLimitResponse(rateLimitResult);
|
|
}
|
|
|
|
try {
|
|
// Try to get from cache first
|
|
const cacheKey = cacheKeys.stats();
|
|
const cached = await getCached<StatsData>(cacheKey);
|
|
if (cached) {
|
|
return NextResponse.json(cached, {
|
|
headers: {
|
|
'X-Cache': 'HIT',
|
|
'Cache-Control': 'public, max-age=3600, stale-while-revalidate=7200',
|
|
...createRateLimitHeaders(rateLimitResult),
|
|
},
|
|
});
|
|
}
|
|
|
|
// Browse-ready filter: exclude duplicates and aggregators
|
|
const browseReady = sql`${skills.isDuplicate} = false AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`;
|
|
|
|
// Consolidate all skill stats into a single query (browse-ready only)
|
|
const statsResult = await db
|
|
.select({
|
|
totalSkills: sql<number>`count(*)::int`,
|
|
totalDownloads: sql<number>`coalesce(sum(${skills.downloadCount}), 0)::int`,
|
|
totalContributors: sql<number>`count(distinct ${skills.githubOwner})::int`,
|
|
})
|
|
.from(skills)
|
|
.where(browseReady);
|
|
|
|
// Get category count in separate query (different table)
|
|
const categoryResult = await db
|
|
.select({ count: sql<number>`count(*)::int` })
|
|
.from(categories);
|
|
|
|
const stats = statsResult[0];
|
|
const totalCategories = categoryResult[0]?.count ?? 0;
|
|
|
|
const data: StatsData = {
|
|
totalSkills: stats?.totalSkills ?? 0,
|
|
totalDownloads: stats?.totalDownloads ?? 0,
|
|
totalCategories,
|
|
totalContributors: stats?.totalContributors ?? 0,
|
|
platforms: 5, // Claude, Codex, Copilot, Cursor, Windsurf
|
|
};
|
|
|
|
// Cache the result
|
|
await setCache(cacheKey, data, cacheTTL.stats);
|
|
|
|
return NextResponse.json(data, {
|
|
headers: {
|
|
'X-Cache': 'MISS',
|
|
'Cache-Control': 'public, max-age=3600, stale-while-revalidate=7200',
|
|
...createRateLimitHeaders(rateLimitResult),
|
|
},
|
|
});
|
|
} catch (error) {
|
|
console.error('Error fetching stats:', error);
|
|
return NextResponse.json(
|
|
{ error: 'Failed to fetch stats' },
|
|
{ status: 500 }
|
|
);
|
|
}
|
|
}
|