sync: add AI review pipeline, parallel skill-files fetch, Redis cache improvements
- feat(review): add skill_reviews table, review API endpoints (pending/submit/stats), admin auth - perf: parallel file fetching in skill-files API (was sequential → timeout) - fix: handle Date serialization from Redis cache - fix: align curation batch scripts with current browseReadyFilter Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
4
.gitignore
vendored
4
.gitignore
vendored
@@ -43,6 +43,10 @@ drizzle/*.sql
|
||||
*.dump
|
||||
data/backups/
|
||||
|
||||
# Curation session data (review batches/results contain raw_content)
|
||||
scripts/curation/data/*
|
||||
!scripts/curation/data/.gitkeep
|
||||
|
||||
# Claude Code
|
||||
.claude/skills/
|
||||
.claude/settings.local.json
|
||||
|
||||
@@ -20,10 +20,11 @@ interface NewSkillsPageProps {
|
||||
}
|
||||
|
||||
// Format date to "X hours/days ago" with locale support
|
||||
function formatTimeAgo(date: Date | null, locale: string): string {
|
||||
function formatTimeAgo(date: Date | string | null, locale: string): string {
|
||||
if (!date) return locale === 'fa' ? 'اخیراً' : 'Recently';
|
||||
const d = date instanceof Date ? date : new Date(date);
|
||||
const now = new Date();
|
||||
const diffMs = now.getTime() - date.getTime();
|
||||
const diffMs = now.getTime() - d.getTime();
|
||||
const diffHours = Math.floor(diffMs / (1000 * 60 * 60));
|
||||
const diffDays = Math.floor(diffHours / 24);
|
||||
|
||||
|
||||
@@ -118,7 +118,7 @@ export default async function SkillPage({ params }: SkillPageProps) {
|
||||
securityStatus: dbSkill.securityStatus || 'pass',
|
||||
isVerified: dbSkill.isVerified || false,
|
||||
createdAt: dbSkill.createdAt,
|
||||
updatedAt: dbSkill.updatedAt ? dbSkill.updatedAt.toLocaleDateString(locale === 'fa' ? 'fa-IR' : 'en-US') : 'N/A',
|
||||
updatedAt: dbSkill.updatedAt ? new Date(dbSkill.updatedAt).toLocaleDateString(locale === 'fa' ? 'fa-IR' : 'en-US') : 'N/A',
|
||||
rating: dbSkill.rating || 0,
|
||||
ratingCount: dbSkill.ratingCount || 0,
|
||||
sourceFormat: dbSkill.sourceFormat || 'skill.md',
|
||||
|
||||
142
apps/web/app/api/review/pending/route.ts
Normal file
142
apps/web/app/api/review/pending/route.ts
Normal file
@@ -0,0 +1,142 @@
|
||||
import { type NextRequest, NextResponse } from 'next/server';
|
||||
import { createDb, skillReviewQueries } from '@skillhub/db';
|
||||
import { requireAdmin } from '@/lib/admin-auth';
|
||||
import { withRateLimit, createRateLimitResponse, createRateLimitHeaders } from '@/lib/rate-limit';
|
||||
|
||||
export const maxDuration = 60; // seconds — queries 55K+ skills with owner-cap logic
|
||||
|
||||
const db = createDb();
|
||||
|
||||
/**
|
||||
* GET /api/review/pending
|
||||
* Returns a batch of skills ready for AI review.
|
||||
* Supports owner-capped batches for diversity and hybrid re-review/new-review mixing.
|
||||
*
|
||||
* Query params:
|
||||
* batch_size - number of skills to return (default 20, max 50)
|
||||
* offset - number of skills to skip for pagination (default 0)
|
||||
* min_quality - minimum quality_score (default 50)
|
||||
* security - security_status filter (default "pass")
|
||||
* priority - "re-review" to show needs-re-review first, "re-review-all" to include already ai-reviewed skills
|
||||
* owner_limit - max skills per github_owner in batch (default 0=unlimited, max 10)
|
||||
*/
|
||||
export async function GET(request: NextRequest) {
|
||||
// Rate limiting
|
||||
const rateLimitResult = await withRateLimit(request, 'anonymous');
|
||||
if (!rateLimitResult.allowed) {
|
||||
return createRateLimitResponse(rateLimitResult);
|
||||
}
|
||||
|
||||
// Admin check (supports API key auth via Authorization header)
|
||||
const adminCheck = await requireAdmin(request);
|
||||
if (!adminCheck.authorized) {
|
||||
return adminCheck.response;
|
||||
}
|
||||
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const batchSize = Math.min(
|
||||
Math.max(parseInt(searchParams.get('batch_size') ?? '20', 10) || 20, 1),
|
||||
50
|
||||
);
|
||||
const offset = Math.max(parseInt(searchParams.get('offset') ?? '0', 10) || 0, 0);
|
||||
const minQuality = parseInt(searchParams.get('min_quality') ?? '50', 10) || 50;
|
||||
const securityPass = searchParams.get('security') !== 'any';
|
||||
const priorityParam = searchParams.get('priority') ?? '';
|
||||
const priorityReReview = priorityParam === 're-review';
|
||||
const reReviewAll = priorityParam === 're-review-all';
|
||||
const ownerLimit = Math.min(
|
||||
Math.max(parseInt(searchParams.get('owner_limit') ?? '0', 10) || 0, 0),
|
||||
10
|
||||
);
|
||||
|
||||
// Run counts in parallel
|
||||
const [totalPending, reReviews] = await Promise.all([
|
||||
skillReviewQueries.countPending(db, { minQuality, securityPass }),
|
||||
skillReviewQueries.countReReviews(db),
|
||||
]);
|
||||
|
||||
// Re-review-all mode: skip hybrid mixing, just fetch all reviewable skills
|
||||
let batch: Array<{ id: string; githubOwner?: string; github_owner?: string; [key: string]: unknown }> = [];
|
||||
|
||||
if (reReviewAll) {
|
||||
const extraSlots = ownerLimit > 0 ? Math.min(batchSize, 10) : 0;
|
||||
const allBatch = await skillReviewQueries.getPending(db, {
|
||||
batchSize: batchSize + extraSlots,
|
||||
offset,
|
||||
minQuality,
|
||||
securityPass,
|
||||
reReviewAll: true,
|
||||
ownerLimit,
|
||||
}) as typeof batch;
|
||||
batch = [...allBatch].slice(0, batchSize);
|
||||
} else {
|
||||
// Normal mode: Hybrid batch — mix re-reviews (up to 5) with new reviews
|
||||
const reReviewSlots = Math.min(5, reReviews, batchSize);
|
||||
|
||||
if (reReviewSlots > 0 && !priorityReReview) {
|
||||
// Fetch re-reviews first (up to 5)
|
||||
const reReviewBatch = await skillReviewQueries.getPending(db, {
|
||||
batchSize: reReviewSlots,
|
||||
minQuality: 0, // re-reviews regardless of quality
|
||||
securityPass,
|
||||
priorityReReview: true,
|
||||
ownerLimit,
|
||||
});
|
||||
batch = [...reReviewBatch] as typeof batch;
|
||||
}
|
||||
|
||||
// Fill remaining slots with new reviews (or all slots if priority=re-review)
|
||||
const remainingSlots = batchSize - batch.length;
|
||||
if (remainingSlots > 0) {
|
||||
// Request extra to compensate for owner deduplication
|
||||
const extraSlots = ownerLimit > 0 ? Math.min(remainingSlots, 10) : 0;
|
||||
const newBatch = await skillReviewQueries.getPending(db, {
|
||||
batchSize: priorityReReview ? batchSize : (remainingSlots + extraSlots),
|
||||
offset,
|
||||
minQuality,
|
||||
securityPass,
|
||||
priorityReReview,
|
||||
ownerLimit,
|
||||
}) as typeof batch;
|
||||
|
||||
if (priorityReReview) {
|
||||
batch = [...newBatch];
|
||||
} else if (ownerLimit > 0 && batch.length > 0) {
|
||||
// Deduplicate owners: count per-owner across both batches
|
||||
const ownerCounts: Record<string, number> = {};
|
||||
for (const s of batch) {
|
||||
const owner = (s.githubOwner ?? s.github_owner ?? 'unknown') as string;
|
||||
ownerCounts[owner] = (ownerCounts[owner] || 0) + 1;
|
||||
}
|
||||
for (const s of newBatch) {
|
||||
const owner = (s.githubOwner ?? s.github_owner ?? 'unknown') as string;
|
||||
if ((ownerCounts[owner] || 0) >= ownerLimit) continue; // Skip — owner already at cap
|
||||
ownerCounts[owner] = (ownerCounts[owner] || 0) + 1;
|
||||
batch.push(s);
|
||||
if (batch.length >= batchSize) break;
|
||||
}
|
||||
} else {
|
||||
batch = [...batch, ...newBatch].slice(0, batchSize);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return NextResponse.json(
|
||||
{
|
||||
total_pending: totalPending,
|
||||
re_reviews: reReviews,
|
||||
batch,
|
||||
},
|
||||
{
|
||||
headers: createRateLimitHeaders(rateLimitResult),
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('[Review] Error fetching pending:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch pending reviews' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
79
apps/web/app/api/review/stats/route.ts
Normal file
79
apps/web/app/api/review/stats/route.ts
Normal file
@@ -0,0 +1,79 @@
|
||||
import { type NextRequest, NextResponse } from 'next/server';
|
||||
import { createDb, skillReviewQueries } from '@skillhub/db';
|
||||
import { requireAdmin } from '@/lib/admin-auth';
|
||||
import { withRateLimit, createRateLimitResponse, createRateLimitHeaders } from '@/lib/rate-limit';
|
||||
import { getCached, setCache, cacheKeys, cacheTTL } from '@/lib/cache';
|
||||
|
||||
const db = createDb();
|
||||
|
||||
interface ReviewStatsData {
|
||||
unreviewed: number;
|
||||
auto_scored: number;
|
||||
ai_reviewed: number;
|
||||
verified: number;
|
||||
needs_re_review: number;
|
||||
total_reviews: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /api/review/stats
|
||||
* Returns review pipeline status statistics.
|
||||
*/
|
||||
export async function GET(request: NextRequest) {
|
||||
// Rate limiting
|
||||
const rateLimitResult = await withRateLimit(request, 'anonymous');
|
||||
if (!rateLimitResult.allowed) {
|
||||
return createRateLimitResponse(rateLimitResult);
|
||||
}
|
||||
|
||||
// Admin check (supports API key auth via Authorization header)
|
||||
const adminCheck = await requireAdmin(request);
|
||||
if (!adminCheck.authorized) {
|
||||
return adminCheck.response;
|
||||
}
|
||||
|
||||
try {
|
||||
// Check cache
|
||||
const cacheKey = cacheKeys.reviewStats();
|
||||
const cached = await getCached<ReviewStatsData>(cacheKey);
|
||||
if (cached) {
|
||||
return NextResponse.json(cached, {
|
||||
headers: {
|
||||
'X-Cache': 'HIT',
|
||||
...createRateLimitHeaders(rateLimitResult),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Run stats and total reviews count in parallel
|
||||
const [statusCounts, totalReviews] = await Promise.all([
|
||||
skillReviewQueries.getStats(db),
|
||||
skillReviewQueries.countTotalReviews(db),
|
||||
]);
|
||||
|
||||
const data: ReviewStatsData = {
|
||||
unreviewed: statusCounts['unreviewed'] ?? 0,
|
||||
auto_scored: statusCounts['auto-scored'] ?? 0,
|
||||
ai_reviewed: statusCounts['ai-reviewed'] ?? 0,
|
||||
verified: statusCounts['verified'] ?? 0,
|
||||
needs_re_review: statusCounts['needs-re-review'] ?? 0,
|
||||
total_reviews: totalReviews,
|
||||
};
|
||||
|
||||
// Cache for 60 seconds
|
||||
await setCache(cacheKey, data, cacheTTL.reviewStats);
|
||||
|
||||
return NextResponse.json(data, {
|
||||
headers: {
|
||||
'X-Cache': 'MISS',
|
||||
...createRateLimitHeaders(rateLimitResult),
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[Review] Error fetching stats:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch review stats' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
151
apps/web/app/api/review/submit/route.ts
Normal file
151
apps/web/app/api/review/submit/route.ts
Normal file
@@ -0,0 +1,151 @@
|
||||
import { type NextRequest, NextResponse } from 'next/server';
|
||||
import { createDb, skillReviewQueries } from '@skillhub/db';
|
||||
import { requireAdmin } from '@/lib/admin-auth';
|
||||
import { withRateLimit, createRateLimitResponse, createRateLimitHeaders } from '@/lib/rate-limit';
|
||||
|
||||
export const maxDuration = 60; // seconds — batch DB writes can be slow
|
||||
|
||||
const db = createDb();
|
||||
|
||||
interface ReviewItem {
|
||||
skill_id: string;
|
||||
ai_score?: number;
|
||||
instruction_quality?: number;
|
||||
description_precision?: number;
|
||||
usefulness?: number;
|
||||
technical_soundness?: number;
|
||||
review_notes?: string;
|
||||
suggested_categories?: string[];
|
||||
blog_worthy?: boolean;
|
||||
collection_candidate?: string | null;
|
||||
needs_improvement?: string | null;
|
||||
i18n_priority?: number;
|
||||
content_hash_at_review?: string;
|
||||
set_verified?: boolean;
|
||||
}
|
||||
|
||||
function validateReviews(body: unknown): { reviews: ReviewItem[] } | { error: string } {
|
||||
if (!body || typeof body !== 'object' || !('reviews' in body)) {
|
||||
return { error: 'Missing "reviews" array in request body' };
|
||||
}
|
||||
|
||||
const { reviews } = body as { reviews: unknown };
|
||||
if (!Array.isArray(reviews) || reviews.length === 0) {
|
||||
return { error: '"reviews" must be a non-empty array' };
|
||||
}
|
||||
if (reviews.length > 50) {
|
||||
return { error: '"reviews" array cannot exceed 50 items' };
|
||||
}
|
||||
|
||||
for (let i = 0; i < reviews.length; i++) {
|
||||
const r = reviews[i];
|
||||
if (!r || typeof r !== 'object') {
|
||||
return { error: `reviews[${i}] is not an object` };
|
||||
}
|
||||
const item = r as Record<string, unknown>;
|
||||
if (typeof item.skill_id !== 'string' || item.skill_id.length === 0) {
|
||||
return { error: `reviews[${i}].skill_id is required and must be a non-empty string` };
|
||||
}
|
||||
// Validate score fields (0-100 integers, optional)
|
||||
for (const field of ['ai_score', 'instruction_quality', 'description_precision', 'usefulness', 'technical_soundness']) {
|
||||
if (item[field] !== undefined && item[field] !== null) {
|
||||
if (typeof item[field] !== 'number' || !Number.isInteger(item[field]) || (item[field] as number) < 0 || (item[field] as number) > 100) {
|
||||
return { error: `reviews[${i}].${field} must be an integer 0-100` };
|
||||
}
|
||||
}
|
||||
}
|
||||
// Validate i18n_priority (0-2)
|
||||
if (item.i18n_priority !== undefined && item.i18n_priority !== null) {
|
||||
if (typeof item.i18n_priority !== 'number' || !Number.isInteger(item.i18n_priority) || item.i18n_priority < 0 || item.i18n_priority > 2) {
|
||||
return { error: `reviews[${i}].i18n_priority must be an integer 0-2` };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { reviews: reviews as ReviewItem[] };
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/review/submit
|
||||
* Submit AI review results for a batch of skills.
|
||||
*/
|
||||
export async function POST(request: NextRequest) {
|
||||
// Rate limiting
|
||||
const rateLimitResult = await withRateLimit(request, 'anonymous');
|
||||
if (!rateLimitResult.allowed) {
|
||||
return createRateLimitResponse(rateLimitResult);
|
||||
}
|
||||
|
||||
// Admin check (supports API key auth via Authorization header)
|
||||
const adminCheck = await requireAdmin(request);
|
||||
if (!adminCheck.authorized) {
|
||||
return adminCheck.response;
|
||||
}
|
||||
|
||||
// Primary server check
|
||||
const isPrimary = process.env.IS_PRIMARY_SERVER !== 'false';
|
||||
if (!isPrimary) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Write operations only on primary server' },
|
||||
{ status: 503 }
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const body = await request.json();
|
||||
const result = validateReviews(body);
|
||||
if ('error' in result) {
|
||||
return NextResponse.json(
|
||||
{ error: result.error },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const { reviews } = result;
|
||||
|
||||
// Insert review rows
|
||||
const dbReviews = reviews.map((r) => ({
|
||||
skillId: r.skill_id,
|
||||
reviewer: 'claude-code' as const,
|
||||
aiScore: r.ai_score,
|
||||
instructionQuality: r.instruction_quality,
|
||||
descriptionPrecision: r.description_precision,
|
||||
usefulness: r.usefulness,
|
||||
technicalSoundness: r.technical_soundness,
|
||||
reviewNotes: r.review_notes,
|
||||
suggestedCategories: r.suggested_categories,
|
||||
blogWorthy: r.blog_worthy,
|
||||
collectionCandidate: r.collection_candidate ?? undefined,
|
||||
needsImprovement: r.needs_improvement ?? undefined,
|
||||
i18nPriority: r.i18n_priority,
|
||||
contentHashAtReview: r.content_hash_at_review,
|
||||
}));
|
||||
|
||||
await skillReviewQueries.createBatch(db, dbReviews);
|
||||
|
||||
// Update review_status on each skill
|
||||
let verifiedCount = 0;
|
||||
for (const r of reviews) {
|
||||
const newStatus = r.set_verified ? 'verified' : 'ai-reviewed';
|
||||
if (r.set_verified) verifiedCount++;
|
||||
|
||||
await skillReviewQueries.updateSkillReviewStatus(db, r.skill_id, newStatus);
|
||||
}
|
||||
|
||||
return NextResponse.json(
|
||||
{
|
||||
submitted: reviews.length,
|
||||
verified: verifiedCount,
|
||||
},
|
||||
{
|
||||
headers: createRateLimitHeaders(rateLimitResult),
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('[Review] Error submitting reviews:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to submit reviews' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -9,8 +9,17 @@ export const maxDuration = 60; // 60 seconds
|
||||
// Maximum recursion depth to prevent infinite loops
|
||||
const MAX_DEPTH = 5;
|
||||
|
||||
// Maximum number of files to fetch per skill
|
||||
const MAX_FILES = 50;
|
||||
|
||||
// Maximum total content size (2MB) to prevent huge responses
|
||||
const MAX_TOTAL_SIZE = 2 * 1024 * 1024;
|
||||
|
||||
// Concurrency limit for parallel file content fetches
|
||||
const PARALLEL_FETCH_LIMIT = 5;
|
||||
|
||||
// Fetch timeout in milliseconds
|
||||
const FETCH_TIMEOUT = 30000; // 30 seconds
|
||||
const FETCH_TIMEOUT = 15000; // 15 seconds per individual fetch
|
||||
|
||||
// Create database connection
|
||||
const db = createDb();
|
||||
@@ -209,42 +218,56 @@ export async function GET(request: NextRequest) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively fetch all files in a skill folder from GitHub
|
||||
* @param depth - Current recursion depth (max MAX_DEPTH levels)
|
||||
* @param headers - GitHub API headers (with token rotation)
|
||||
* @param token - Current token for stats tracking
|
||||
* Helper: run async tasks with concurrency limit
|
||||
*/
|
||||
async function fetchSkillFiles(
|
||||
async function parallelLimit<T>(
|
||||
tasks: (() => Promise<T>)[],
|
||||
limit: number
|
||||
): Promise<T[]> {
|
||||
const results: T[] = [];
|
||||
let idx = 0;
|
||||
|
||||
async function runNext(): Promise<void> {
|
||||
while (idx < tasks.length) {
|
||||
const currentIdx = idx++;
|
||||
results[currentIdx] = await tasks[currentIdx]();
|
||||
}
|
||||
}
|
||||
|
||||
const workers = Array.from({ length: Math.min(limit, tasks.length) }, () => runNext());
|
||||
await Promise.all(workers);
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively collect file metadata (directory listings only, no content fetch).
|
||||
* Returns flat list of file entries to fetch content for.
|
||||
*/
|
||||
async function collectFileEntries(
|
||||
owner: string,
|
||||
repo: string,
|
||||
path: string,
|
||||
ref: string,
|
||||
depth: number = 0,
|
||||
depth: number,
|
||||
headers: Record<string, string>,
|
||||
token: string | null
|
||||
): Promise<SkillFile[]> {
|
||||
// Prevent infinite recursion
|
||||
): Promise<Array<{ item: GitHubFile; relativePath: string }>> {
|
||||
if (depth > MAX_DEPTH) {
|
||||
console.warn(`Max depth (${MAX_DEPTH}) reached at: ${path}`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const files: SkillFile[] = [];
|
||||
|
||||
// Fetch directory contents with timeout
|
||||
const apiUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${path}?ref=${ref}`;
|
||||
const response = await fetch(apiUrl, {
|
||||
headers,
|
||||
signal: AbortSignal.timeout(FETCH_TIMEOUT),
|
||||
});
|
||||
|
||||
// Update token stats for rotation
|
||||
if (token) {
|
||||
await updateTokenStats(token, response.headers);
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
// Check for rate limiting
|
||||
if (response.status === 403) {
|
||||
const rateLimitRemaining = response.headers.get('x-ratelimit-remaining');
|
||||
if (rateLimitRemaining === '0') {
|
||||
@@ -255,60 +278,92 @@ async function fetchSkillFiles(
|
||||
}
|
||||
|
||||
const contents: GitHubFile[] = await response.json();
|
||||
const entries: Array<{ item: GitHubFile; relativePath: string }> = [];
|
||||
|
||||
// Process each item
|
||||
for (const item of contents) {
|
||||
const relativePath = item.path.replace(`${path}/`, '').replace(path, '') || item.name;
|
||||
|
||||
if (item.type === 'file') {
|
||||
// Determine if file is binary
|
||||
const isBinary = !isTextFile(item.name);
|
||||
|
||||
if (!isBinary && item.size < 1024 * 1024) {
|
||||
// For text files (< 1MB), fetch content
|
||||
try {
|
||||
const content = await fetchFileContent(owner, repo, item.path, ref, headers, token);
|
||||
files.push({
|
||||
name: item.name,
|
||||
path: item.path.replace(`${path}/`, '').replace(path, '') || item.name,
|
||||
content,
|
||||
size: item.size,
|
||||
isBinary: false,
|
||||
});
|
||||
} catch {
|
||||
// If content fetch fails, mark as binary (will use download URL)
|
||||
files.push({
|
||||
name: item.name,
|
||||
path: item.path.replace(`${path}/`, '').replace(path, '') || item.name,
|
||||
content: '',
|
||||
size: item.size,
|
||||
isBinary: true,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// For binary or large files, don't store content (use download URL)
|
||||
files.push({
|
||||
name: item.name,
|
||||
path: item.path.replace(`${path}/`, '').replace(path, '') || item.name,
|
||||
content: '',
|
||||
size: item.size,
|
||||
isBinary: true,
|
||||
});
|
||||
}
|
||||
entries.push({ item, relativePath });
|
||||
} else if (item.type === 'dir') {
|
||||
// Recursively fetch subdirectory (with depth limit)
|
||||
const subPath = `${path}/${item.name}`;
|
||||
const subFiles = await fetchSkillFiles(owner, repo, subPath, ref, depth + 1, headers, token);
|
||||
|
||||
// Add subdirectory files with proper paths
|
||||
for (const subFile of subFiles) {
|
||||
files.push({
|
||||
...subFile,
|
||||
path: `${item.name}/${subFile.path}`,
|
||||
const subEntries = await collectFileEntries(
|
||||
owner, repo, `${path}/${item.name}`, ref, depth + 1, headers, token
|
||||
);
|
||||
for (const sub of subEntries) {
|
||||
entries.push({
|
||||
item: sub.item,
|
||||
relativePath: `${item.name}/${sub.relativePath}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return files;
|
||||
return entries;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch all files in a skill folder from GitHub.
|
||||
* Phase 1: Collect directory tree (sequential, required for recursion)
|
||||
* Phase 2: Fetch file contents in parallel (fast, with concurrency limit)
|
||||
*/
|
||||
async function fetchSkillFiles(
|
||||
owner: string,
|
||||
repo: string,
|
||||
path: string,
|
||||
ref: string,
|
||||
depth: number = 0,
|
||||
headers: Record<string, string>,
|
||||
token: string | null
|
||||
): Promise<SkillFile[]> {
|
||||
// Phase 1: Collect all file entries from directory tree
|
||||
const entries = await collectFileEntries(owner, repo, path, ref, depth, headers, token);
|
||||
|
||||
// Apply MAX_FILES limit
|
||||
if (entries.length > MAX_FILES) {
|
||||
console.warn(`[skill-files] Skill has ${entries.length} files, limiting to ${MAX_FILES}`);
|
||||
entries.length = MAX_FILES;
|
||||
}
|
||||
|
||||
// Phase 2: Fetch file contents in parallel
|
||||
let totalSize = 0;
|
||||
const tasks = entries.map((entry) => async (): Promise<SkillFile> => {
|
||||
const { item, relativePath } = entry;
|
||||
const isBinary = !isTextFile(item.name);
|
||||
|
||||
if (!isBinary && item.size < 1024 * 1024 && totalSize + item.size <= MAX_TOTAL_SIZE) {
|
||||
try {
|
||||
const content = await fetchFileContent(owner, repo, item.path, ref, headers, token);
|
||||
totalSize += item.size;
|
||||
return {
|
||||
name: item.name,
|
||||
path: relativePath,
|
||||
content,
|
||||
size: item.size,
|
||||
isBinary: false,
|
||||
};
|
||||
} catch {
|
||||
// Content fetch failed — return as binary with download URL
|
||||
return {
|
||||
name: item.name,
|
||||
path: relativePath,
|
||||
content: '',
|
||||
size: item.size,
|
||||
isBinary: true,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Binary, too large, or total size exceeded
|
||||
return {
|
||||
name: item.name,
|
||||
path: relativePath,
|
||||
content: '',
|
||||
size: item.size,
|
||||
isBinary: true,
|
||||
};
|
||||
});
|
||||
|
||||
return parallelLimit(tasks, PARALLEL_FETCH_LIMIT);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -22,14 +22,14 @@ interface SkillCardProps {
|
||||
securityStatus: string | null;
|
||||
isVerified: boolean | null;
|
||||
sourceFormat?: string | null;
|
||||
createdAt?: Date | null;
|
||||
updatedAt?: Date | null;
|
||||
createdAt?: Date | string | null;
|
||||
updatedAt?: Date | string | null;
|
||||
};
|
||||
locale: string;
|
||||
/** Show time badge (for New Skills page) */
|
||||
showTimeBadge?: 'created' | 'updated' | null;
|
||||
/** Format time as "X days ago" */
|
||||
formatTimeAgo?: (date: Date | null, locale: string) => string;
|
||||
formatTimeAgo?: (date: Date | string | null, locale: string) => string;
|
||||
}
|
||||
|
||||
export function SkillCard({ skill, locale, showTimeBadge, formatTimeAgo }: SkillCardProps) {
|
||||
|
||||
46
apps/web/lib/admin-auth.ts
Normal file
46
apps/web/lib/admin-auth.ts
Normal file
@@ -0,0 +1,46 @@
|
||||
import { type NextRequest, NextResponse } from 'next/server';
|
||||
import { auth } from '@/lib/auth';
|
||||
import { createDb, userQueries } from '@skillhub/db';
|
||||
|
||||
/**
|
||||
* Check if the current request is from an admin user.
|
||||
* Supports two auth methods:
|
||||
* 1. API key via Authorization: Bearer <REVIEW_API_KEY> header (for automation/scripts)
|
||||
* 2. Session-based admin check (for dashboard)
|
||||
*
|
||||
* Pass the request object to enable API key auth.
|
||||
* Returns the user on success, or a NextResponse error to return early.
|
||||
*/
|
||||
export async function requireAdmin(request?: NextRequest): Promise<
|
||||
| { authorized: true; username: string }
|
||||
| { authorized: false; response: NextResponse }
|
||||
> {
|
||||
// Check API key auth first (for automation/scripting)
|
||||
if (request) {
|
||||
const authHeader = request.headers.get('authorization');
|
||||
const apiKey = process.env.REVIEW_API_KEY;
|
||||
if (apiKey && authHeader === `Bearer ${apiKey}`) {
|
||||
return { authorized: true, username: 'api-key' };
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to session auth
|
||||
const session = await auth();
|
||||
if (!session?.user?.githubId) {
|
||||
return {
|
||||
authorized: false,
|
||||
response: NextResponse.json({ error: 'Unauthorized' }, { status: 401 }),
|
||||
};
|
||||
}
|
||||
|
||||
const db = createDb();
|
||||
const user = await userQueries.getByGithubId(db, session.user.githubId);
|
||||
if (!user?.isAdmin) {
|
||||
return {
|
||||
authorized: false,
|
||||
response: NextResponse.json({ error: 'Admin access required' }, { status: 403 }),
|
||||
};
|
||||
}
|
||||
|
||||
return { authorized: true, username: user.username };
|
||||
}
|
||||
@@ -168,6 +168,7 @@ export const cacheKeys = {
|
||||
skill: (id: string) => `skill:${id.replace(/\//g, ':')}`,
|
||||
skillView: (skillId: string, ip: string) => `view:${skillId.replace(/\//g, ':')}:${ip}`,
|
||||
skillDownload: (skillId: string, ip: string) => `download:${skillId.replace(/\//g, ':')}:${ip}`,
|
||||
reviewStats: () => 'review:stats',
|
||||
};
|
||||
|
||||
// TTL values in seconds
|
||||
@@ -184,6 +185,7 @@ export const cacheTTL = {
|
||||
pageCount: 60 * 60, // 1 hour
|
||||
view: 60 * 60, // 1 hour - same IP can only count as 1 view per hour
|
||||
download: 5 * 60, // 5 minutes - same IP can only count as 1 download per 5 min
|
||||
reviewStats: 60, // 1 minute - admin review stats
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
removalRequests,
|
||||
addRequests,
|
||||
emailSubscriptions,
|
||||
skillReviews,
|
||||
} from './schema.js';
|
||||
|
||||
import type * as schema from './schema.js';
|
||||
@@ -642,6 +643,10 @@ export const skillQueries = {
|
||||
updatedAt: new Date(),
|
||||
// Clear cachedFiles if commitSha changed (on insert, it's null anyway)
|
||||
cachedFiles: shouldClearCache ? null : undefined,
|
||||
// Auto-score new skills when crawler provides both security and quality
|
||||
reviewStatus: (skill.securityStatus && skill.qualityScore != null)
|
||||
? 'auto-scored'
|
||||
: skill.reviewStatus,
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: skills.id,
|
||||
@@ -672,9 +677,17 @@ export const skillQueries = {
|
||||
// which only updates it when content-related columns actually change
|
||||
// Clear cachedFiles if commitSha changed
|
||||
...(shouldClearCache ? { cachedFiles: null } : {}),
|
||||
// Re-review mechanism (Phase 6.4): mark previously reviewed skills for re-review
|
||||
// when their content changes. Only affects 'verified' and 'ai-reviewed' skills.
|
||||
...(shouldTriggerReReview ? { reviewStatus: 'needs-re-review' } : {}),
|
||||
// Review status management:
|
||||
// 1. Re-review (Phase 6.4): mark previously reviewed skills for re-review
|
||||
// when their content changes. Only affects 'verified' and 'ai-reviewed' skills.
|
||||
// 2. Auto-score: when crawler provides both securityStatus and qualityScore,
|
||||
// set reviewStatus='auto-scored' so skill enters the review pipeline.
|
||||
// Only for new/unreviewed skills (don't overwrite existing review status).
|
||||
...(shouldTriggerReReview
|
||||
? { reviewStatus: 'needs-re-review' }
|
||||
: (skill.securityStatus && skill.qualityScore != null)
|
||||
? { reviewStatus: sql`CASE WHEN ${skills.reviewStatus} IS NULL OR ${skills.reviewStatus} = 'unreviewed' THEN 'auto-scored' ELSE ${skills.reviewStatus} END` }
|
||||
: {}),
|
||||
},
|
||||
})
|
||||
.returning();
|
||||
@@ -2245,3 +2258,368 @@ export const emailSubscriptionQueries = {
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* Skill review queries (AI review pipeline — Phase 7)
|
||||
*/
|
||||
export const skillReviewQueries = {
|
||||
/**
|
||||
* Submit a review for a skill (insert new review row)
|
||||
*/
|
||||
create: async (
|
||||
db: DB,
|
||||
data: {
|
||||
skillId: string;
|
||||
reviewer?: string;
|
||||
aiScore?: number;
|
||||
instructionQuality?: number;
|
||||
descriptionPrecision?: number;
|
||||
usefulness?: number;
|
||||
technicalSoundness?: number;
|
||||
reviewNotes?: string;
|
||||
suggestedCategories?: string[];
|
||||
blogWorthy?: boolean;
|
||||
collectionCandidate?: string;
|
||||
needsImprovement?: string;
|
||||
i18nPriority?: number;
|
||||
contentHashAtReview?: string;
|
||||
reviewVersion?: number;
|
||||
}
|
||||
) => {
|
||||
const result = await db
|
||||
.insert(skillReviews)
|
||||
.values({
|
||||
skillId: data.skillId,
|
||||
reviewer: data.reviewer ?? 'claude-code',
|
||||
aiScore: data.aiScore,
|
||||
instructionQuality: data.instructionQuality,
|
||||
descriptionPrecision: data.descriptionPrecision,
|
||||
usefulness: data.usefulness,
|
||||
technicalSoundness: data.technicalSoundness,
|
||||
reviewNotes: data.reviewNotes,
|
||||
suggestedCategories: data.suggestedCategories,
|
||||
blogWorthy: data.blogWorthy ?? false,
|
||||
collectionCandidate: data.collectionCandidate,
|
||||
needsImprovement: data.needsImprovement,
|
||||
i18nPriority: data.i18nPriority ?? 0,
|
||||
contentHashAtReview: data.contentHashAtReview,
|
||||
reviewVersion: data.reviewVersion ?? 1,
|
||||
})
|
||||
.returning();
|
||||
return result[0];
|
||||
},
|
||||
|
||||
/**
|
||||
* Submit multiple reviews in a batch
|
||||
*/
|
||||
createBatch: async (
|
||||
db: DB,
|
||||
reviews: Array<{
|
||||
skillId: string;
|
||||
reviewer?: string;
|
||||
aiScore?: number;
|
||||
instructionQuality?: number;
|
||||
descriptionPrecision?: number;
|
||||
usefulness?: number;
|
||||
technicalSoundness?: number;
|
||||
reviewNotes?: string;
|
||||
suggestedCategories?: string[];
|
||||
blogWorthy?: boolean;
|
||||
collectionCandidate?: string;
|
||||
needsImprovement?: string;
|
||||
i18nPriority?: number;
|
||||
contentHashAtReview?: string;
|
||||
reviewVersion?: number;
|
||||
}>
|
||||
) => {
|
||||
return db
|
||||
.insert(skillReviews)
|
||||
.values(
|
||||
reviews.map((r) => ({
|
||||
skillId: r.skillId,
|
||||
reviewer: r.reviewer ?? 'claude-code',
|
||||
aiScore: r.aiScore,
|
||||
instructionQuality: r.instructionQuality,
|
||||
descriptionPrecision: r.descriptionPrecision,
|
||||
usefulness: r.usefulness,
|
||||
technicalSoundness: r.technicalSoundness,
|
||||
reviewNotes: r.reviewNotes,
|
||||
suggestedCategories: r.suggestedCategories,
|
||||
blogWorthy: r.blogWorthy ?? false,
|
||||
collectionCandidate: r.collectionCandidate,
|
||||
needsImprovement: r.needsImprovement,
|
||||
i18nPriority: r.i18nPriority ?? 0,
|
||||
contentHashAtReview: r.contentHashAtReview,
|
||||
reviewVersion: r.reviewVersion ?? 1,
|
||||
}))
|
||||
)
|
||||
.returning();
|
||||
},
|
||||
|
||||
/**
|
||||
* Get latest review for a skill
|
||||
*/
|
||||
getLatestBySkillId: async (db: DB, skillId: string) => {
|
||||
const result = await db
|
||||
.select()
|
||||
.from(skillReviews)
|
||||
.where(eq(skillReviews.skillId, skillId))
|
||||
.orderBy(desc(skillReviews.reviewedAt))
|
||||
.limit(1);
|
||||
return result[0] ?? null;
|
||||
},
|
||||
|
||||
/**
|
||||
* Get all reviews for a skill (history)
|
||||
*/
|
||||
getAllBySkillId: async (db: DB, skillId: string) => {
|
||||
return db
|
||||
.select()
|
||||
.from(skillReviews)
|
||||
.where(eq(skillReviews.skillId, skillId))
|
||||
.orderBy(desc(skillReviews.reviewedAt));
|
||||
},
|
||||
|
||||
/**
|
||||
* Get review by id
|
||||
*/
|
||||
getById: async (db: DB, id: number) => {
|
||||
const result = await db
|
||||
.select()
|
||||
.from(skillReviews)
|
||||
.where(eq(skillReviews.id, id))
|
||||
.limit(1);
|
||||
return result[0] ?? null;
|
||||
},
|
||||
|
||||
/**
|
||||
* Get blog-worthy skills (join with skills for details)
|
||||
*/
|
||||
getBlogCandidates: async (db: DB, limit = 50) => {
|
||||
return db
|
||||
.select({
|
||||
reviewId: skillReviews.id,
|
||||
skillId: skillReviews.skillId,
|
||||
aiScore: skillReviews.aiScore,
|
||||
reviewNotes: skillReviews.reviewNotes,
|
||||
suggestedCategories: skillReviews.suggestedCategories,
|
||||
skillName: skills.name,
|
||||
skillDescription: skills.description,
|
||||
githubStars: skills.githubStars,
|
||||
reviewStatus: skills.reviewStatus,
|
||||
})
|
||||
.from(skillReviews)
|
||||
.innerJoin(skills, eq(skillReviews.skillId, skills.id))
|
||||
.where(eq(skillReviews.blogWorthy, true))
|
||||
.orderBy(desc(skillReviews.aiScore))
|
||||
.limit(limit);
|
||||
},
|
||||
|
||||
/**
|
||||
* Get review stats (counts by review_status)
|
||||
*/
|
||||
getStats: async (db: DB) => {
|
||||
const result = await db
|
||||
.select({
|
||||
reviewStatus: skills.reviewStatus,
|
||||
count: sql<number>`count(*)::int`,
|
||||
})
|
||||
.from(skills)
|
||||
.where(browseReadyFilter)
|
||||
.groupBy(skills.reviewStatus);
|
||||
|
||||
const stats: Record<string, number> = {};
|
||||
for (const row of result) {
|
||||
stats[row.reviewStatus ?? 'unreviewed'] = row.count;
|
||||
}
|
||||
return stats;
|
||||
},
|
||||
|
||||
/**
|
||||
* Count skills pending AI review (for total_pending in API response)
|
||||
* Only counts standard SKILL.md format skills
|
||||
*/
|
||||
countPending: async (
|
||||
db: DB,
|
||||
options: { minQuality?: number; securityPass?: boolean } = {}
|
||||
) => {
|
||||
const { minQuality = 50, securityPass = true } = options;
|
||||
|
||||
const conditions = [
|
||||
browseReadyFilter,
|
||||
eq(skills.isBlocked, false),
|
||||
eq(skills.reviewStatus, 'auto-scored'),
|
||||
eq(skills.sourceFormat, 'skill.md'),
|
||||
// Pre-filters via safe SQL function (handles invalid UTF-8 gracefully)
|
||||
eq(skills.isDeprecated, false),
|
||||
sql`raw_content_passes_prefilter(${skills.rawContent})`,
|
||||
];
|
||||
if (minQuality > 0) conditions.push(gte(skills.qualityScore, minQuality));
|
||||
if (securityPass) conditions.push(eq(skills.securityStatus, 'pass'));
|
||||
|
||||
const result = await db
|
||||
.select({ count: sql<number>`count(*)::int` })
|
||||
.from(skills)
|
||||
.where(and(...conditions));
|
||||
return result[0]?.count ?? 0;
|
||||
},
|
||||
|
||||
/**
|
||||
* Count skills needing re-review
|
||||
*/
|
||||
countReReviews: async (db: DB) => {
|
||||
const result = await db
|
||||
.select({ count: sql<number>`count(*)::int` })
|
||||
.from(skills)
|
||||
.where(
|
||||
and(
|
||||
browseReadyFilter,
|
||||
eq(skills.isBlocked, false),
|
||||
eq(skills.reviewStatus, 'needs-re-review'),
|
||||
// Pre-filters via safe SQL function (handles invalid UTF-8 gracefully)
|
||||
eq(skills.sourceFormat, 'skill.md'),
|
||||
eq(skills.isDeprecated, false),
|
||||
sql`raw_content_passes_prefilter(${skills.rawContent})`,
|
||||
)
|
||||
);
|
||||
return result[0]?.count ?? 0;
|
||||
},
|
||||
|
||||
/**
|
||||
* Count total review rows in skill_reviews table
|
||||
*/
|
||||
countTotalReviews: async (db: DB) => {
|
||||
const result = await db
|
||||
.select({ count: sql<number>`count(*)::int` })
|
||||
.from(skillReviews);
|
||||
return result[0]?.count ?? 0;
|
||||
},
|
||||
|
||||
/**
|
||||
* Get skills pending AI review (auto-scored, security=pass, quality>=threshold)
|
||||
* Only returns standard SKILL.md format skills (excludes agents.md, cursorrules, etc.)
|
||||
* Supports owner-capped batches to ensure diversity (max N skills per github_owner).
|
||||
* Pre-filters: excludes auto-generated docs, internal paths, very short content, deprecated.
|
||||
*/
|
||||
getPending: async (
|
||||
db: DB,
|
||||
options: {
|
||||
batchSize?: number;
|
||||
offset?: number;
|
||||
minQuality?: number;
|
||||
securityPass?: boolean;
|
||||
priorityReReview?: boolean;
|
||||
reReviewAll?: boolean;
|
||||
ownerLimit?: number;
|
||||
} = {}
|
||||
) => {
|
||||
const {
|
||||
batchSize = 20,
|
||||
offset = 0,
|
||||
minQuality = 50,
|
||||
securityPass = true,
|
||||
priorityReReview = false,
|
||||
reReviewAll = false,
|
||||
ownerLimit = 0,
|
||||
} = options;
|
||||
|
||||
const conditions = [
|
||||
browseReadyFilter,
|
||||
eq(skills.isBlocked, false),
|
||||
// Only standard SKILL.md format — exclude agents.md, cursorrules, windsurfrules, copilot-instructions
|
||||
eq(skills.sourceFormat, 'skill.md'),
|
||||
// Skip deprecated skills
|
||||
eq(skills.isDeprecated, false),
|
||||
// Pre-filters via safe SQL function (handles invalid UTF-8 gracefully)
|
||||
sql`raw_content_passes_prefilter(${skills.rawContent})`,
|
||||
];
|
||||
|
||||
if (reReviewAll) {
|
||||
// Re-review mode: include already-reviewed skills (for rubric version upgrades)
|
||||
conditions.push(
|
||||
sql`${skills.reviewStatus} IN ('ai-reviewed', 'needs-re-review', 'auto-scored')`
|
||||
);
|
||||
} else if (priorityReReview) {
|
||||
conditions.push(eq(skills.reviewStatus, 'needs-re-review'));
|
||||
} else {
|
||||
conditions.push(eq(skills.reviewStatus, 'auto-scored'));
|
||||
}
|
||||
|
||||
if (minQuality > 0) {
|
||||
conditions.push(gte(skills.qualityScore, minQuality));
|
||||
}
|
||||
|
||||
if (securityPass) {
|
||||
conditions.push(eq(skills.securityStatus, 'pass'));
|
||||
}
|
||||
|
||||
const selectFields = {
|
||||
id: skills.id,
|
||||
name: skills.name,
|
||||
description: skills.description,
|
||||
rawContent: skills.rawContent,
|
||||
qualityScore: skills.qualityScore,
|
||||
securityStatus: skills.securityStatus,
|
||||
githubStars: skills.githubStars,
|
||||
skillType: skills.skillType,
|
||||
reviewStatus: skills.reviewStatus,
|
||||
contentHash: skills.contentHash,
|
||||
githubOwner: skills.githubOwner,
|
||||
githubRepo: skills.githubRepo,
|
||||
skillPath: skills.skillPath,
|
||||
branch: skills.branch,
|
||||
};
|
||||
|
||||
// Owner-capped batch: limit skills per github_owner for diversity
|
||||
if (ownerLimit > 0) {
|
||||
const whereClause = and(...conditions);
|
||||
const results = await db.execute(sql`
|
||||
WITH ranked AS (
|
||||
SELECT id, name, description, raw_content, quality_score, security_status,
|
||||
github_stars, skill_type, review_status, content_hash,
|
||||
github_owner, github_repo, skill_path, branch,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY github_owner
|
||||
ORDER BY quality_score DESC NULLS LAST, github_stars DESC NULLS LAST
|
||||
) AS owner_rank
|
||||
FROM skills
|
||||
WHERE ${whereClause}
|
||||
)
|
||||
SELECT id, name, description, raw_content AS "rawContent",
|
||||
quality_score AS "qualityScore", security_status AS "securityStatus",
|
||||
github_stars AS "githubStars", skill_type AS "skillType",
|
||||
review_status AS "reviewStatus", content_hash AS "contentHash",
|
||||
github_owner AS "githubOwner", github_repo AS "githubRepo",
|
||||
skill_path AS "skillPath", branch
|
||||
FROM ranked
|
||||
WHERE owner_rank <= ${ownerLimit}
|
||||
ORDER BY quality_score DESC NULLS LAST, github_stars DESC NULLS LAST
|
||||
LIMIT ${batchSize}
|
||||
OFFSET ${offset}
|
||||
`);
|
||||
return [...results];
|
||||
}
|
||||
|
||||
return db
|
||||
.select(selectFields)
|
||||
.from(skills)
|
||||
.where(and(...conditions))
|
||||
.orderBy(desc(skills.qualityScore), desc(skills.githubStars))
|
||||
.limit(batchSize)
|
||||
.offset(offset);
|
||||
},
|
||||
|
||||
/**
|
||||
* Update review_status on a skill
|
||||
*/
|
||||
updateSkillReviewStatus: async (
|
||||
db: DB,
|
||||
skillId: string,
|
||||
reviewStatus: string
|
||||
) => {
|
||||
return db
|
||||
.update(skills)
|
||||
.set({ reviewStatus })
|
||||
.where(eq(skills.id, skillId));
|
||||
},
|
||||
};
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
text,
|
||||
timestamp,
|
||||
integer,
|
||||
serial,
|
||||
jsonb,
|
||||
boolean,
|
||||
index,
|
||||
@@ -65,6 +66,7 @@ export const skills = pgTable(
|
||||
isVerified: boolean('is_verified').default(false),
|
||||
isFeatured: boolean('is_featured').default(false),
|
||||
isBlocked: boolean('is_blocked').default(false), // Blocked from re-indexing (owner requested removal)
|
||||
isDeprecated: boolean('is_deprecated').default(false), // Auto-detected from raw_content (DEPRECATED/ARCHIVED markers)
|
||||
lastScanned: timestamp('last_scanned'),
|
||||
|
||||
// Curation (populated by batch scripts, not crawler)
|
||||
@@ -389,12 +391,52 @@ export const addRequests = pgTable(
|
||||
})
|
||||
);
|
||||
|
||||
/**
|
||||
* AI review results for skills (Layer 2 — verification pipeline)
|
||||
*/
|
||||
export const skillReviews = pgTable(
|
||||
'skill_reviews',
|
||||
{
|
||||
id: serial('id').primaryKey(),
|
||||
skillId: text('skill_id')
|
||||
.references(() => skills.id, { onDelete: 'cascade' })
|
||||
.notNull(),
|
||||
reviewer: text('reviewer').notNull().default('claude-code'), // 'claude-code' | 'admin'
|
||||
|
||||
// Scores (per Skills Guide criteria)
|
||||
aiScore: integer('ai_score'), // 0-100 overall
|
||||
instructionQuality: integer('instruction_quality'), // clarity and structure
|
||||
descriptionPrecision: integer('description_precision'), // description + triggers
|
||||
usefulness: integer('usefulness'), // real-world value
|
||||
technicalSoundness: integer('technical_soundness'), // correctness of commands/APIs
|
||||
|
||||
// Discoveries during review
|
||||
reviewNotes: text('review_notes'),
|
||||
suggestedCategories: jsonb('suggested_categories').$type<string[]>(),
|
||||
blogWorthy: boolean('blog_worthy').default(false),
|
||||
collectionCandidate: text('collection_candidate'),
|
||||
needsImprovement: text('needs_improvement'),
|
||||
i18nPriority: integer('i18n_priority').default(0), // 0=normal, 1=worth translating, 2=high priority
|
||||
|
||||
// Tracking
|
||||
contentHashAtReview: text('content_hash_at_review'), // skill hash at time of review
|
||||
reviewedAt: timestamp('reviewed_at').defaultNow(),
|
||||
reviewVersion: integer('review_version').default(1), // criteria version for recalibration
|
||||
},
|
||||
(table) => ({
|
||||
skillIdx: index('idx_reviews_skill').on(table.skillId),
|
||||
scoreIdx: index('idx_reviews_score').on(table.aiScore),
|
||||
blogIdx: index('idx_reviews_blog').on(table.blogWorthy),
|
||||
})
|
||||
);
|
||||
|
||||
// Relations
|
||||
export const skillsRelations = relations(skills, ({ many }) => ({
|
||||
categories: many(skillCategories),
|
||||
ratings: many(ratings),
|
||||
installations: many(installations),
|
||||
favorites: many(favorites),
|
||||
reviews: many(skillReviews),
|
||||
}));
|
||||
|
||||
export const categoriesRelations = relations(categories, ({ many, one }) => ({
|
||||
@@ -470,6 +512,13 @@ export const addRequestsRelations = relations(addRequests, ({ one }) => ({
|
||||
}),
|
||||
}));
|
||||
|
||||
export const skillReviewsRelations = relations(skillReviews, ({ one }) => ({
|
||||
skill: one(skills, {
|
||||
fields: [skillReviews.skillId],
|
||||
references: [skills.id],
|
||||
}),
|
||||
}));
|
||||
|
||||
/**
|
||||
* Email subscriptions for newsletter and marketing emails
|
||||
*/
|
||||
|
||||
@@ -285,7 +285,6 @@ async function main() {
|
||||
FROM skills
|
||||
WHERE is_blocked = false
|
||||
AND is_duplicate = false
|
||||
AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection'))
|
||||
AND quality_score IS NULL
|
||||
AND raw_content IS NOT NULL
|
||||
`);
|
||||
@@ -305,7 +304,7 @@ async function main() {
|
||||
FROM skills
|
||||
WHERE is_blocked = false
|
||||
AND is_duplicate = false
|
||||
AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection'))
|
||||
AND is_duplicate = false
|
||||
AND quality_score IS NULL
|
||||
AND raw_content IS NOT NULL
|
||||
ORDER BY github_stars DESC NULLS LAST
|
||||
@@ -334,7 +333,7 @@ async function main() {
|
||||
FROM skills
|
||||
WHERE is_blocked = false
|
||||
AND is_duplicate = false
|
||||
AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection'))
|
||||
AND is_duplicate = false
|
||||
AND quality_score IS NULL
|
||||
AND raw_content IS NOT NULL
|
||||
ORDER BY github_stars DESC NULLS LAST
|
||||
|
||||
@@ -114,7 +114,6 @@ async function main() {
|
||||
FROM skills
|
||||
WHERE is_blocked = false
|
||||
AND is_duplicate = false
|
||||
AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection'))
|
||||
AND security_status IS NULL
|
||||
AND raw_content IS NOT NULL
|
||||
`);
|
||||
@@ -135,7 +134,7 @@ async function main() {
|
||||
FROM skills
|
||||
WHERE is_blocked = false
|
||||
AND is_duplicate = false
|
||||
AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection'))
|
||||
AND is_duplicate = false
|
||||
AND security_status IS NULL
|
||||
AND raw_content IS NOT NULL
|
||||
LIMIT 5
|
||||
@@ -163,7 +162,7 @@ async function main() {
|
||||
FROM skills
|
||||
WHERE is_blocked = false
|
||||
AND is_duplicate = false
|
||||
AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection'))
|
||||
AND is_duplicate = false
|
||||
AND security_status IS NULL
|
||||
AND raw_content IS NOT NULL
|
||||
ORDER BY github_stars DESC NULLS LAST
|
||||
|
||||
@@ -483,9 +483,80 @@ CREATE INDEX IF NOT EXISTS idx_skills_content_hash ON skills(content_hash);
|
||||
ALTER TABLE skills ADD COLUMN IF NOT EXISTS review_status TEXT DEFAULT 'unreviewed';
|
||||
CREATE INDEX IF NOT EXISTS idx_skills_review_status ON skills(review_status);
|
||||
|
||||
-- Deprecated detection (Phase 7.4 - February 2026)
|
||||
ALTER TABLE skills ADD COLUMN IF NOT EXISTS is_deprecated BOOLEAN DEFAULT FALSE;
|
||||
|
||||
-- Safe pre-filter function for raw_content (handles invalid UTF-8 gracefully)
|
||||
-- Used by review pipeline queries (getPending, countPending, countReReviews)
|
||||
CREATE OR REPLACE FUNCTION raw_content_passes_prefilter(content TEXT) RETURNS BOOLEAN AS $$
|
||||
BEGIN
|
||||
IF octet_length(content) < 200 THEN RETURN FALSE; END IF;
|
||||
IF position('<!-- generated' in content) > 0 THEN RETURN FALSE; END IF;
|
||||
IF position('/Users/' in LEFT(content, 1000)) > 0 THEN RETURN FALSE; END IF;
|
||||
IF position('C:\Users\' in LEFT(content, 1000)) > 0 THEN RETURN FALSE; END IF;
|
||||
RETURN TRUE;
|
||||
EXCEPTION WHEN OTHERS THEN
|
||||
RETURN FALSE;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql IMMUTABLE;
|
||||
|
||||
-- Batch populate is_deprecated for existing rows (one-time migration)
|
||||
-- Uses PL/pgSQL with exception handling to skip rows with invalid UTF-8
|
||||
DO $$
|
||||
DECLARE
|
||||
r RECORD;
|
||||
rc TEXT;
|
||||
cnt INT := 0;
|
||||
BEGIN
|
||||
FOR r IN SELECT id, raw_content FROM skills WHERE is_deprecated = false LOOP
|
||||
BEGIN
|
||||
rc := LEFT(r.raw_content, 1000);
|
||||
IF rc ~* '(DEPRECATED|ARCHIVED|NO LONGER MAINTAINED|THIS PROJECT IS ABANDONED)' THEN
|
||||
UPDATE skills SET is_deprecated = true WHERE id = r.id;
|
||||
cnt := cnt + 1;
|
||||
END IF;
|
||||
EXCEPTION WHEN OTHERS THEN
|
||||
NULL; -- Skip rows with encoding errors
|
||||
END;
|
||||
END LOOP;
|
||||
RAISE NOTICE 'Marked % skills as deprecated', cnt;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- Repo creation date for duplicate tie-breaking (T074b)
|
||||
ALTER TABLE skills ADD COLUMN IF NOT EXISTS repo_created_at TIMESTAMP WITH TIME ZONE;
|
||||
|
||||
-- AI review results table (Phase 7.1 - February 2026)
|
||||
CREATE TABLE IF NOT EXISTS skill_reviews (
|
||||
id SERIAL PRIMARY KEY,
|
||||
skill_id TEXT NOT NULL REFERENCES skills(id) ON DELETE CASCADE,
|
||||
reviewer TEXT NOT NULL DEFAULT 'claude-code', -- 'claude-code' | 'admin'
|
||||
|
||||
-- Scores (per Skills Guide criteria)
|
||||
ai_score INTEGER, -- 0-100 overall
|
||||
instruction_quality INTEGER, -- clarity and structure
|
||||
description_precision INTEGER, -- description + triggers
|
||||
usefulness INTEGER, -- real-world value
|
||||
technical_soundness INTEGER, -- correctness of commands/APIs
|
||||
|
||||
-- Discoveries during review
|
||||
review_notes TEXT,
|
||||
suggested_categories JSONB, -- text[] stored as JSONB for Drizzle compatibility
|
||||
blog_worthy BOOLEAN DEFAULT FALSE,
|
||||
collection_candidate TEXT,
|
||||
needs_improvement TEXT,
|
||||
i18n_priority INTEGER DEFAULT 0, -- 0=normal, 1=worth translating, 2=high priority
|
||||
|
||||
-- Tracking
|
||||
content_hash_at_review TEXT, -- skill hash at time of review
|
||||
reviewed_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
|
||||
review_version INTEGER DEFAULT 1 -- criteria version for recalibration
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_reviews_skill ON skill_reviews(skill_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_reviews_score ON skill_reviews(ai_score);
|
||||
CREATE INDEX IF NOT EXISTS idx_reviews_blog ON skill_reviews(blog_worthy) WHERE blog_worthy = true;
|
||||
|
||||
-- Grant permissions
|
||||
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO postgres;
|
||||
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO postgres;
|
||||
|
||||
Reference in New Issue
Block a user