sync: add AI review pipeline, parallel skill-files fetch, Redis cache improvements

- feat(review): add skill_reviews table, review API endpoints (pending/submit/stats), admin auth
- perf: parallel file fetching in skill-files API (was sequential → timeout)
- fix: handle Date serialization from Redis cache
- fix: align curation batch scripts with current browseReadyFilter

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
airano
2026-02-23 19:51:43 +03:30
parent 328520bb70
commit 447f9eff59
15 changed files with 1050 additions and 74 deletions

View File

@@ -20,10 +20,11 @@ interface NewSkillsPageProps {
}
// Format date to "X hours/days ago" with locale support
function formatTimeAgo(date: Date | null, locale: string): string {
function formatTimeAgo(date: Date | string | null, locale: string): string {
if (!date) return locale === 'fa' ? 'اخیراً' : 'Recently';
const d = date instanceof Date ? date : new Date(date);
const now = new Date();
const diffMs = now.getTime() - date.getTime();
const diffMs = now.getTime() - d.getTime();
const diffHours = Math.floor(diffMs / (1000 * 60 * 60));
const diffDays = Math.floor(diffHours / 24);

View File

@@ -118,7 +118,7 @@ export default async function SkillPage({ params }: SkillPageProps) {
securityStatus: dbSkill.securityStatus || 'pass',
isVerified: dbSkill.isVerified || false,
createdAt: dbSkill.createdAt,
updatedAt: dbSkill.updatedAt ? dbSkill.updatedAt.toLocaleDateString(locale === 'fa' ? 'fa-IR' : 'en-US') : 'N/A',
updatedAt: dbSkill.updatedAt ? new Date(dbSkill.updatedAt).toLocaleDateString(locale === 'fa' ? 'fa-IR' : 'en-US') : 'N/A',
rating: dbSkill.rating || 0,
ratingCount: dbSkill.ratingCount || 0,
sourceFormat: dbSkill.sourceFormat || 'skill.md',

View File

@@ -0,0 +1,142 @@
import { type NextRequest, NextResponse } from 'next/server';
import { createDb, skillReviewQueries } from '@skillhub/db';
import { requireAdmin } from '@/lib/admin-auth';
import { withRateLimit, createRateLimitResponse, createRateLimitHeaders } from '@/lib/rate-limit';
export const maxDuration = 60; // seconds — queries 55K+ skills with owner-cap logic
const db = createDb();
/**
* GET /api/review/pending
* Returns a batch of skills ready for AI review.
* Supports owner-capped batches for diversity and hybrid re-review/new-review mixing.
*
* Query params:
* batch_size - number of skills to return (default 20, max 50)
* offset - number of skills to skip for pagination (default 0)
* min_quality - minimum quality_score (default 50)
* security - security_status filter (default "pass")
* priority - "re-review" to show needs-re-review first, "re-review-all" to include already ai-reviewed skills
* owner_limit - max skills per github_owner in batch (default 0=unlimited, max 10)
*/
export async function GET(request: NextRequest) {
// Rate limiting
const rateLimitResult = await withRateLimit(request, 'anonymous');
if (!rateLimitResult.allowed) {
return createRateLimitResponse(rateLimitResult);
}
// Admin check (supports API key auth via Authorization header)
const adminCheck = await requireAdmin(request);
if (!adminCheck.authorized) {
return adminCheck.response;
}
try {
const { searchParams } = new URL(request.url);
const batchSize = Math.min(
Math.max(parseInt(searchParams.get('batch_size') ?? '20', 10) || 20, 1),
50
);
const offset = Math.max(parseInt(searchParams.get('offset') ?? '0', 10) || 0, 0);
const minQuality = parseInt(searchParams.get('min_quality') ?? '50', 10) || 50;
const securityPass = searchParams.get('security') !== 'any';
const priorityParam = searchParams.get('priority') ?? '';
const priorityReReview = priorityParam === 're-review';
const reReviewAll = priorityParam === 're-review-all';
const ownerLimit = Math.min(
Math.max(parseInt(searchParams.get('owner_limit') ?? '0', 10) || 0, 0),
10
);
// Run counts in parallel
const [totalPending, reReviews] = await Promise.all([
skillReviewQueries.countPending(db, { minQuality, securityPass }),
skillReviewQueries.countReReviews(db),
]);
// Re-review-all mode: skip hybrid mixing, just fetch all reviewable skills
let batch: Array<{ id: string; githubOwner?: string; github_owner?: string; [key: string]: unknown }> = [];
if (reReviewAll) {
const extraSlots = ownerLimit > 0 ? Math.min(batchSize, 10) : 0;
const allBatch = await skillReviewQueries.getPending(db, {
batchSize: batchSize + extraSlots,
offset,
minQuality,
securityPass,
reReviewAll: true,
ownerLimit,
}) as typeof batch;
batch = [...allBatch].slice(0, batchSize);
} else {
// Normal mode: Hybrid batch — mix re-reviews (up to 5) with new reviews
const reReviewSlots = Math.min(5, reReviews, batchSize);
if (reReviewSlots > 0 && !priorityReReview) {
// Fetch re-reviews first (up to 5)
const reReviewBatch = await skillReviewQueries.getPending(db, {
batchSize: reReviewSlots,
minQuality: 0, // re-reviews regardless of quality
securityPass,
priorityReReview: true,
ownerLimit,
});
batch = [...reReviewBatch] as typeof batch;
}
// Fill remaining slots with new reviews (or all slots if priority=re-review)
const remainingSlots = batchSize - batch.length;
if (remainingSlots > 0) {
// Request extra to compensate for owner deduplication
const extraSlots = ownerLimit > 0 ? Math.min(remainingSlots, 10) : 0;
const newBatch = await skillReviewQueries.getPending(db, {
batchSize: priorityReReview ? batchSize : (remainingSlots + extraSlots),
offset,
minQuality,
securityPass,
priorityReReview,
ownerLimit,
}) as typeof batch;
if (priorityReReview) {
batch = [...newBatch];
} else if (ownerLimit > 0 && batch.length > 0) {
// Deduplicate owners: count per-owner across both batches
const ownerCounts: Record<string, number> = {};
for (const s of batch) {
const owner = (s.githubOwner ?? s.github_owner ?? 'unknown') as string;
ownerCounts[owner] = (ownerCounts[owner] || 0) + 1;
}
for (const s of newBatch) {
const owner = (s.githubOwner ?? s.github_owner ?? 'unknown') as string;
if ((ownerCounts[owner] || 0) >= ownerLimit) continue; // Skip — owner already at cap
ownerCounts[owner] = (ownerCounts[owner] || 0) + 1;
batch.push(s);
if (batch.length >= batchSize) break;
}
} else {
batch = [...batch, ...newBatch].slice(0, batchSize);
}
}
}
return NextResponse.json(
{
total_pending: totalPending,
re_reviews: reReviews,
batch,
},
{
headers: createRateLimitHeaders(rateLimitResult),
}
);
} catch (error) {
console.error('[Review] Error fetching pending:', error);
return NextResponse.json(
{ error: 'Failed to fetch pending reviews' },
{ status: 500 }
);
}
}

View File

@@ -0,0 +1,79 @@
import { type NextRequest, NextResponse } from 'next/server';
import { createDb, skillReviewQueries } from '@skillhub/db';
import { requireAdmin } from '@/lib/admin-auth';
import { withRateLimit, createRateLimitResponse, createRateLimitHeaders } from '@/lib/rate-limit';
import { getCached, setCache, cacheKeys, cacheTTL } from '@/lib/cache';
const db = createDb();
interface ReviewStatsData {
unreviewed: number;
auto_scored: number;
ai_reviewed: number;
verified: number;
needs_re_review: number;
total_reviews: number;
}
/**
* GET /api/review/stats
* Returns review pipeline status statistics.
*/
export async function GET(request: NextRequest) {
// Rate limiting
const rateLimitResult = await withRateLimit(request, 'anonymous');
if (!rateLimitResult.allowed) {
return createRateLimitResponse(rateLimitResult);
}
// Admin check (supports API key auth via Authorization header)
const adminCheck = await requireAdmin(request);
if (!adminCheck.authorized) {
return adminCheck.response;
}
try {
// Check cache
const cacheKey = cacheKeys.reviewStats();
const cached = await getCached<ReviewStatsData>(cacheKey);
if (cached) {
return NextResponse.json(cached, {
headers: {
'X-Cache': 'HIT',
...createRateLimitHeaders(rateLimitResult),
},
});
}
// Run stats and total reviews count in parallel
const [statusCounts, totalReviews] = await Promise.all([
skillReviewQueries.getStats(db),
skillReviewQueries.countTotalReviews(db),
]);
const data: ReviewStatsData = {
unreviewed: statusCounts['unreviewed'] ?? 0,
auto_scored: statusCounts['auto-scored'] ?? 0,
ai_reviewed: statusCounts['ai-reviewed'] ?? 0,
verified: statusCounts['verified'] ?? 0,
needs_re_review: statusCounts['needs-re-review'] ?? 0,
total_reviews: totalReviews,
};
// Cache for 60 seconds
await setCache(cacheKey, data, cacheTTL.reviewStats);
return NextResponse.json(data, {
headers: {
'X-Cache': 'MISS',
...createRateLimitHeaders(rateLimitResult),
},
});
} catch (error) {
console.error('[Review] Error fetching stats:', error);
return NextResponse.json(
{ error: 'Failed to fetch review stats' },
{ status: 500 }
);
}
}

View File

@@ -0,0 +1,151 @@
import { type NextRequest, NextResponse } from 'next/server';
import { createDb, skillReviewQueries } from '@skillhub/db';
import { requireAdmin } from '@/lib/admin-auth';
import { withRateLimit, createRateLimitResponse, createRateLimitHeaders } from '@/lib/rate-limit';
export const maxDuration = 60; // seconds — batch DB writes can be slow
const db = createDb();
interface ReviewItem {
skill_id: string;
ai_score?: number;
instruction_quality?: number;
description_precision?: number;
usefulness?: number;
technical_soundness?: number;
review_notes?: string;
suggested_categories?: string[];
blog_worthy?: boolean;
collection_candidate?: string | null;
needs_improvement?: string | null;
i18n_priority?: number;
content_hash_at_review?: string;
set_verified?: boolean;
}
function validateReviews(body: unknown): { reviews: ReviewItem[] } | { error: string } {
if (!body || typeof body !== 'object' || !('reviews' in body)) {
return { error: 'Missing "reviews" array in request body' };
}
const { reviews } = body as { reviews: unknown };
if (!Array.isArray(reviews) || reviews.length === 0) {
return { error: '"reviews" must be a non-empty array' };
}
if (reviews.length > 50) {
return { error: '"reviews" array cannot exceed 50 items' };
}
for (let i = 0; i < reviews.length; i++) {
const r = reviews[i];
if (!r || typeof r !== 'object') {
return { error: `reviews[${i}] is not an object` };
}
const item = r as Record<string, unknown>;
if (typeof item.skill_id !== 'string' || item.skill_id.length === 0) {
return { error: `reviews[${i}].skill_id is required and must be a non-empty string` };
}
// Validate score fields (0-100 integers, optional)
for (const field of ['ai_score', 'instruction_quality', 'description_precision', 'usefulness', 'technical_soundness']) {
if (item[field] !== undefined && item[field] !== null) {
if (typeof item[field] !== 'number' || !Number.isInteger(item[field]) || (item[field] as number) < 0 || (item[field] as number) > 100) {
return { error: `reviews[${i}].${field} must be an integer 0-100` };
}
}
}
// Validate i18n_priority (0-2)
if (item.i18n_priority !== undefined && item.i18n_priority !== null) {
if (typeof item.i18n_priority !== 'number' || !Number.isInteger(item.i18n_priority) || item.i18n_priority < 0 || item.i18n_priority > 2) {
return { error: `reviews[${i}].i18n_priority must be an integer 0-2` };
}
}
}
return { reviews: reviews as ReviewItem[] };
}
/**
* POST /api/review/submit
* Submit AI review results for a batch of skills.
*/
export async function POST(request: NextRequest) {
// Rate limiting
const rateLimitResult = await withRateLimit(request, 'anonymous');
if (!rateLimitResult.allowed) {
return createRateLimitResponse(rateLimitResult);
}
// Admin check (supports API key auth via Authorization header)
const adminCheck = await requireAdmin(request);
if (!adminCheck.authorized) {
return adminCheck.response;
}
// Primary server check
const isPrimary = process.env.IS_PRIMARY_SERVER !== 'false';
if (!isPrimary) {
return NextResponse.json(
{ error: 'Write operations only on primary server' },
{ status: 503 }
);
}
try {
const body = await request.json();
const result = validateReviews(body);
if ('error' in result) {
return NextResponse.json(
{ error: result.error },
{ status: 400 }
);
}
const { reviews } = result;
// Insert review rows
const dbReviews = reviews.map((r) => ({
skillId: r.skill_id,
reviewer: 'claude-code' as const,
aiScore: r.ai_score,
instructionQuality: r.instruction_quality,
descriptionPrecision: r.description_precision,
usefulness: r.usefulness,
technicalSoundness: r.technical_soundness,
reviewNotes: r.review_notes,
suggestedCategories: r.suggested_categories,
blogWorthy: r.blog_worthy,
collectionCandidate: r.collection_candidate ?? undefined,
needsImprovement: r.needs_improvement ?? undefined,
i18nPriority: r.i18n_priority,
contentHashAtReview: r.content_hash_at_review,
}));
await skillReviewQueries.createBatch(db, dbReviews);
// Update review_status on each skill
let verifiedCount = 0;
for (const r of reviews) {
const newStatus = r.set_verified ? 'verified' : 'ai-reviewed';
if (r.set_verified) verifiedCount++;
await skillReviewQueries.updateSkillReviewStatus(db, r.skill_id, newStatus);
}
return NextResponse.json(
{
submitted: reviews.length,
verified: verifiedCount,
},
{
headers: createRateLimitHeaders(rateLimitResult),
}
);
} catch (error) {
console.error('[Review] Error submitting reviews:', error);
return NextResponse.json(
{ error: 'Failed to submit reviews' },
{ status: 500 }
);
}
}

View File

@@ -9,8 +9,17 @@ export const maxDuration = 60; // 60 seconds
// Maximum recursion depth to prevent infinite loops
const MAX_DEPTH = 5;
// Maximum number of files to fetch per skill
const MAX_FILES = 50;
// Maximum total content size (2MB) to prevent huge responses
const MAX_TOTAL_SIZE = 2 * 1024 * 1024;
// Concurrency limit for parallel file content fetches
const PARALLEL_FETCH_LIMIT = 5;
// Fetch timeout in milliseconds
const FETCH_TIMEOUT = 30000; // 30 seconds
const FETCH_TIMEOUT = 15000; // 15 seconds per individual fetch
// Create database connection
const db = createDb();
@@ -209,42 +218,56 @@ export async function GET(request: NextRequest) {
}
/**
* Recursively fetch all files in a skill folder from GitHub
* @param depth - Current recursion depth (max MAX_DEPTH levels)
* @param headers - GitHub API headers (with token rotation)
* @param token - Current token for stats tracking
* Helper: run async tasks with concurrency limit
*/
async function fetchSkillFiles(
async function parallelLimit<T>(
tasks: (() => Promise<T>)[],
limit: number
): Promise<T[]> {
const results: T[] = [];
let idx = 0;
async function runNext(): Promise<void> {
while (idx < tasks.length) {
const currentIdx = idx++;
results[currentIdx] = await tasks[currentIdx]();
}
}
const workers = Array.from({ length: Math.min(limit, tasks.length) }, () => runNext());
await Promise.all(workers);
return results;
}
/**
* Recursively collect file metadata (directory listings only, no content fetch).
* Returns flat list of file entries to fetch content for.
*/
async function collectFileEntries(
owner: string,
repo: string,
path: string,
ref: string,
depth: number = 0,
depth: number,
headers: Record<string, string>,
token: string | null
): Promise<SkillFile[]> {
// Prevent infinite recursion
): Promise<Array<{ item: GitHubFile; relativePath: string }>> {
if (depth > MAX_DEPTH) {
console.warn(`Max depth (${MAX_DEPTH}) reached at: ${path}`);
return [];
}
const files: SkillFile[] = [];
// Fetch directory contents with timeout
const apiUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${path}?ref=${ref}`;
const response = await fetch(apiUrl, {
headers,
signal: AbortSignal.timeout(FETCH_TIMEOUT),
});
// Update token stats for rotation
if (token) {
await updateTokenStats(token, response.headers);
}
if (!response.ok) {
// Check for rate limiting
if (response.status === 403) {
const rateLimitRemaining = response.headers.get('x-ratelimit-remaining');
if (rateLimitRemaining === '0') {
@@ -255,60 +278,92 @@ async function fetchSkillFiles(
}
const contents: GitHubFile[] = await response.json();
const entries: Array<{ item: GitHubFile; relativePath: string }> = [];
// Process each item
for (const item of contents) {
const relativePath = item.path.replace(`${path}/`, '').replace(path, '') || item.name;
if (item.type === 'file') {
// Determine if file is binary
const isBinary = !isTextFile(item.name);
if (!isBinary && item.size < 1024 * 1024) {
// For text files (< 1MB), fetch content
try {
const content = await fetchFileContent(owner, repo, item.path, ref, headers, token);
files.push({
name: item.name,
path: item.path.replace(`${path}/`, '').replace(path, '') || item.name,
content,
size: item.size,
isBinary: false,
});
} catch {
// If content fetch fails, mark as binary (will use download URL)
files.push({
name: item.name,
path: item.path.replace(`${path}/`, '').replace(path, '') || item.name,
content: '',
size: item.size,
isBinary: true,
});
}
} else {
// For binary or large files, don't store content (use download URL)
files.push({
name: item.name,
path: item.path.replace(`${path}/`, '').replace(path, '') || item.name,
content: '',
size: item.size,
isBinary: true,
});
}
entries.push({ item, relativePath });
} else if (item.type === 'dir') {
// Recursively fetch subdirectory (with depth limit)
const subPath = `${path}/${item.name}`;
const subFiles = await fetchSkillFiles(owner, repo, subPath, ref, depth + 1, headers, token);
// Add subdirectory files with proper paths
for (const subFile of subFiles) {
files.push({
...subFile,
path: `${item.name}/${subFile.path}`,
const subEntries = await collectFileEntries(
owner, repo, `${path}/${item.name}`, ref, depth + 1, headers, token
);
for (const sub of subEntries) {
entries.push({
item: sub.item,
relativePath: `${item.name}/${sub.relativePath}`,
});
}
}
}
return files;
return entries;
}
/**
* Fetch all files in a skill folder from GitHub.
* Phase 1: Collect directory tree (sequential, required for recursion)
* Phase 2: Fetch file contents in parallel (fast, with concurrency limit)
*/
async function fetchSkillFiles(
owner: string,
repo: string,
path: string,
ref: string,
depth: number = 0,
headers: Record<string, string>,
token: string | null
): Promise<SkillFile[]> {
// Phase 1: Collect all file entries from directory tree
const entries = await collectFileEntries(owner, repo, path, ref, depth, headers, token);
// Apply MAX_FILES limit
if (entries.length > MAX_FILES) {
console.warn(`[skill-files] Skill has ${entries.length} files, limiting to ${MAX_FILES}`);
entries.length = MAX_FILES;
}
// Phase 2: Fetch file contents in parallel
let totalSize = 0;
const tasks = entries.map((entry) => async (): Promise<SkillFile> => {
const { item, relativePath } = entry;
const isBinary = !isTextFile(item.name);
if (!isBinary && item.size < 1024 * 1024 && totalSize + item.size <= MAX_TOTAL_SIZE) {
try {
const content = await fetchFileContent(owner, repo, item.path, ref, headers, token);
totalSize += item.size;
return {
name: item.name,
path: relativePath,
content,
size: item.size,
isBinary: false,
};
} catch {
// Content fetch failed — return as binary with download URL
return {
name: item.name,
path: relativePath,
content: '',
size: item.size,
isBinary: true,
};
}
}
// Binary, too large, or total size exceeded
return {
name: item.name,
path: relativePath,
content: '',
size: item.size,
isBinary: true,
};
});
return parallelLimit(tasks, PARALLEL_FETCH_LIMIT);
}
/**

View File

@@ -22,14 +22,14 @@ interface SkillCardProps {
securityStatus: string | null;
isVerified: boolean | null;
sourceFormat?: string | null;
createdAt?: Date | null;
updatedAt?: Date | null;
createdAt?: Date | string | null;
updatedAt?: Date | string | null;
};
locale: string;
/** Show time badge (for New Skills page) */
showTimeBadge?: 'created' | 'updated' | null;
/** Format time as "X days ago" */
formatTimeAgo?: (date: Date | null, locale: string) => string;
formatTimeAgo?: (date: Date | string | null, locale: string) => string;
}
export function SkillCard({ skill, locale, showTimeBadge, formatTimeAgo }: SkillCardProps) {

View File

@@ -0,0 +1,46 @@
import { type NextRequest, NextResponse } from 'next/server';
import { auth } from '@/lib/auth';
import { createDb, userQueries } from '@skillhub/db';
/**
* Check if the current request is from an admin user.
* Supports two auth methods:
* 1. API key via Authorization: Bearer <REVIEW_API_KEY> header (for automation/scripts)
* 2. Session-based admin check (for dashboard)
*
* Pass the request object to enable API key auth.
* Returns the user on success, or a NextResponse error to return early.
*/
export async function requireAdmin(request?: NextRequest): Promise<
| { authorized: true; username: string }
| { authorized: false; response: NextResponse }
> {
// Check API key auth first (for automation/scripting)
if (request) {
const authHeader = request.headers.get('authorization');
const apiKey = process.env.REVIEW_API_KEY;
if (apiKey && authHeader === `Bearer ${apiKey}`) {
return { authorized: true, username: 'api-key' };
}
}
// Fall back to session auth
const session = await auth();
if (!session?.user?.githubId) {
return {
authorized: false,
response: NextResponse.json({ error: 'Unauthorized' }, { status: 401 }),
};
}
const db = createDb();
const user = await userQueries.getByGithubId(db, session.user.githubId);
if (!user?.isAdmin) {
return {
authorized: false,
response: NextResponse.json({ error: 'Admin access required' }, { status: 403 }),
};
}
return { authorized: true, username: user.username };
}

View File

@@ -168,6 +168,7 @@ export const cacheKeys = {
skill: (id: string) => `skill:${id.replace(/\//g, ':')}`,
skillView: (skillId: string, ip: string) => `view:${skillId.replace(/\//g, ':')}:${ip}`,
skillDownload: (skillId: string, ip: string) => `download:${skillId.replace(/\//g, ':')}:${ip}`,
reviewStats: () => 'review:stats',
};
// TTL values in seconds
@@ -184,6 +185,7 @@ export const cacheTTL = {
pageCount: 60 * 60, // 1 hour
view: 60 * 60, // 1 hour - same IP can only count as 1 view per hour
download: 5 * 60, // 5 minutes - same IP can only count as 1 download per 5 min
reviewStats: 60, // 1 minute - admin review stats
};
/**