Add comprehensive data curation system to clean up the 197K skill dataset and show only quality browse-ready skills to users. Phase 1 — Database exploration: - Explore scripts (explore.ts, explore.mjs, explore.sql) for analysis - Discovered: 69% duplicates, 77% aggregator/fork noise Phase 2 — Data cleanup and classification: - Schema: 6 new curation columns + 4 indexes - curate.mjs: 8-step pipeline (classify, dedup, fork detection, etc.) - Result: 197K → 60K unique → 16K browse-ready skills - Bug fix: securityStatus was computed but never stored during crawl Phase 3 — UI browse-ready filters: - browseReadyFilter applied to 17+ query functions - Homepage stats show accurate browse-ready counts - Stats API filtered (previously had no WHERE clause) - Category counts recalculated (e.g. 45K → 3.1K) - Featured skills exclude duplicates and aggregators Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
101 lines
3.2 KiB
TypeScript
101 lines
3.2 KiB
TypeScript
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
import { NextRequest } from 'next/server';
|
|
|
|
// Mock rate limiting - must be before route import
|
|
vi.mock('@/lib/rate-limit', () => ({
|
|
withRateLimit: vi.fn().mockResolvedValue({ allowed: true, remaining: 100, limit: 120, resetAt: Date.now() + 60000 }),
|
|
createRateLimitResponse: vi.fn(),
|
|
createRateLimitHeaders: vi.fn().mockReturnValue({}),
|
|
}));
|
|
|
|
// Mock cache - must be before route import
|
|
vi.mock('@/lib/cache', () => ({
|
|
getCached: vi.fn().mockResolvedValue(null),
|
|
setCache: vi.fn().mockResolvedValue(undefined),
|
|
cacheKeys: { stats: () => 'stats' },
|
|
cacheTTL: { stats: 3600 },
|
|
}));
|
|
|
|
// Mock the db module - must be before imports that use it
|
|
vi.mock('@skillhub/db', () => {
|
|
const mockStatsRow = [{ totalSkills: 100, totalDownloads: 5000, totalContributors: 50 }];
|
|
const mockCategoryRow = [{ count: 8 }];
|
|
return {
|
|
createDb: vi.fn(() => ({
|
|
select: vi.fn().mockReturnValue({
|
|
from: vi.fn((table: unknown) => {
|
|
// categories table query returns count directly (no .where())
|
|
if (table === 'categories-table') {
|
|
return Promise.resolve(mockCategoryRow);
|
|
}
|
|
// skills table query has .where() chained
|
|
return {
|
|
where: vi.fn().mockResolvedValue(mockStatsRow),
|
|
};
|
|
}),
|
|
}),
|
|
})),
|
|
skills: { downloadCount: 'download_count', githubOwner: 'github_owner', isDuplicate: 'is_duplicate', skillType: 'skill_type' },
|
|
categories: 'categories-table',
|
|
sql: vi.fn((..._args: unknown[]) => 'mock-sql'),
|
|
};
|
|
});
|
|
|
|
import { GET } from './route';
|
|
|
|
// Helper to create mock request
|
|
function createMockRequest(url = 'http://localhost:3000/api/stats') {
|
|
return new NextRequest(url);
|
|
}
|
|
|
|
describe('GET /api/stats', () => {
|
|
beforeEach(() => {
|
|
vi.clearAllMocks();
|
|
});
|
|
|
|
it('should return totalSkills count', async () => {
|
|
const request = createMockRequest();
|
|
const response = await GET(request);
|
|
const data = await response.json();
|
|
|
|
expect(response.status).toBe(200);
|
|
expect(data.totalSkills).toBeDefined();
|
|
expect(typeof data.totalSkills).toBe('number');
|
|
});
|
|
|
|
it('should return totalDownloads sum', async () => {
|
|
const request = createMockRequest();
|
|
const response = await GET(request);
|
|
const data = await response.json();
|
|
|
|
expect(data.totalDownloads).toBeDefined();
|
|
expect(typeof data.totalDownloads).toBe('number');
|
|
});
|
|
|
|
it('should return totalCategories count', async () => {
|
|
const request = createMockRequest();
|
|
const response = await GET(request);
|
|
const data = await response.json();
|
|
|
|
expect(data.totalCategories).toBeDefined();
|
|
expect(typeof data.totalCategories).toBe('number');
|
|
});
|
|
|
|
it('should return totalContributors count', async () => {
|
|
const request = createMockRequest();
|
|
const response = await GET(request);
|
|
const data = await response.json();
|
|
|
|
expect(data.totalContributors).toBeDefined();
|
|
expect(typeof data.totalContributors).toBe('number');
|
|
});
|
|
|
|
it('should return platforms count', async () => {
|
|
const request = createMockRequest();
|
|
const response = await GET(request);
|
|
const data = await response.json();
|
|
|
|
expect(data.platforms).toBe(5);
|
|
});
|
|
});
|