sync: stale skill detection, sentry filtering, claim removal, review improvements

Stale Detection:
- Detect skills removed/moved from GitHub (3 consecutive 404s threshold)
- Hide stale skills from browse/search, show warning banner on detail page
- Serve cached files with isStale flag when GitHub returns 404
- Add stale-check crawler command for batch verification
- CLI shows warning when installing stale skills from cache

Sentry & Error Handling:
- Filter browser extension errors and add denyUrls
- Anti-inflation measures and sentinel recalibration for curation

Claim & Removal:
- Enhanced ClaimForm with repo-level removal support
- Add repo-removal-request API endpoint with tests
- Improved owner page with bilingual content

Review Pipeline:
- Review version and reviewer tracking in submit API
- Source format filter for pending reviews
- Updated review tests

Other:
- Updated i18n strings (en/fa)
- BrowseFilters improvements
- Dockerfile updates

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
airano
2026-03-12 05:59:26 +03:30
parent 447f9eff59
commit 4212b5ba47
31 changed files with 1590 additions and 58 deletions

View File

@@ -90,10 +90,21 @@ Sanitized: `anthropics/skills/pdf` → `anthropics__skills__pdf`
## Indexer ## Indexer
```bash ```bash
docker compose exec indexer node dist/crawl.js full # Full crawl docker compose exec indexer node dist/crawl.js full # Full crawl
docker compose exec indexer node dist/crawl.js incremental # Last 24h docker compose exec indexer node dist/crawl.js incremental # Last 24h
docker compose exec indexer node dist/crawl.js sync-meili # Sync to Meilisearch docker compose exec indexer node dist/crawl.js sync-meili # Sync to Meilisearch
docker compose exec indexer node dist/crawl.js deep-scan # Scan discovered repos docker compose exec indexer node dist/crawl.js deep-scan # Scan discovered repos
docker compose exec indexer node dist/crawl.js add-repo <owner/repo> # Manually add a repo to DB
docker compose exec indexer node dist/crawl.js process-add-requests # Process user-submitted add requests
```
### Database Migration (container)
```bash
# Add new columns (run when schema changes are deployed)
docker compose exec db psql -U skillhub -d skillhub -c "ALTER TABLE skills ADD COLUMN IF NOT EXISTS is_owner_claimed boolean NOT NULL DEFAULT false;"
docker compose exec db psql -U skillhub -d skillhub -c "ALTER TABLE discovered_repos ADD COLUMN IF NOT EXISTS is_blocked boolean NOT NULL DEFAULT false;"
docker compose exec db psql -U skillhub -d skillhub -c "CREATE INDEX IF NOT EXISTS idx_skills_owner_claimed ON skills(is_owner_claimed) WHERE is_owner_claimed = true;"
docker compose exec db psql -U skillhub -d skillhub -c "CREATE INDEX IF NOT EXISTS idx_discovered_repos_blocked ON discovered_repos(is_blocked) WHERE is_blocked = true;"
``` ```
## Security ## Security

View File

@@ -98,6 +98,14 @@ export async function install(skillId: string, options: InstallOptions): Promise
const cachedFiles = await getSkillFiles(skillInfo.id); const cachedFiles = await getSkillFiles(skillInfo.id);
if (cachedFiles && cachedFiles.files.length > 0) { if (cachedFiles && cachedFiles.files.length > 0) {
// Warn if skill is stale (served from cache because GitHub returned 404)
if (cachedFiles.isStale) {
spinner.warn(chalk.yellow(
'This skill may have been removed from its GitHub repository.\n' +
' Files were served from the SkillHub cache and may be outdated.'
));
spinner.start('Installing from cached files...');
}
// Use sourceFormat from API response if available // Use sourceFormat from API response if available
if (cachedFiles.sourceFormat) { if (cachedFiles.sourceFormat) {
sourceFormat = cachedFiles.sourceFormat as SourceFormat; sourceFormat = cachedFiles.sourceFormat as SourceFormat;

View File

@@ -231,6 +231,8 @@ export interface SkillFilesResponse {
files: SkillFile[]; files: SkillFile[];
fromCache: boolean; fromCache: boolean;
cachedAt?: string; cachedAt?: string;
isStale?: boolean;
staleWarning?: string;
} }
/** /**

View File

@@ -45,6 +45,7 @@ ARG NEXT_PUBLIC_OPENPANEL_API_URL
ARG NEXT_PUBLIC_OPENPANEL_SCRIPT_URL ARG NEXT_PUBLIC_OPENPANEL_SCRIPT_URL
ARG NEXT_PUBLIC_UMAMI_WEBSITE_ID ARG NEXT_PUBLIC_UMAMI_WEBSITE_ID
ARG NEXT_PUBLIC_UMAMI_URL ARG NEXT_PUBLIC_UMAMI_URL
ARG NEXT_PUBLIC_IS_PRIMARY
# Convert ARGs to ENVs so they're available during build # Convert ARGs to ENVs so they're available during build
ENV NEXT_PUBLIC_APP_URL=$NEXT_PUBLIC_APP_URL ENV NEXT_PUBLIC_APP_URL=$NEXT_PUBLIC_APP_URL
@@ -56,6 +57,7 @@ ENV NEXT_PUBLIC_OPENPANEL_API_URL=$NEXT_PUBLIC_OPENPANEL_API_URL
ENV NEXT_PUBLIC_OPENPANEL_SCRIPT_URL=$NEXT_PUBLIC_OPENPANEL_SCRIPT_URL ENV NEXT_PUBLIC_OPENPANEL_SCRIPT_URL=$NEXT_PUBLIC_OPENPANEL_SCRIPT_URL
ENV NEXT_PUBLIC_UMAMI_WEBSITE_ID=$NEXT_PUBLIC_UMAMI_WEBSITE_ID ENV NEXT_PUBLIC_UMAMI_WEBSITE_ID=$NEXT_PUBLIC_UMAMI_WEBSITE_ID
ENV NEXT_PUBLIC_UMAMI_URL=$NEXT_PUBLIC_UMAMI_URL ENV NEXT_PUBLIC_UMAMI_URL=$NEXT_PUBLIC_UMAMI_URL
ENV NEXT_PUBLIC_IS_PRIMARY=$NEXT_PUBLIC_IS_PRIMARY
# Build the application (turbo will build dependencies first) # Build the application (turbo will build dependencies first)
RUN pnpm turbo run build --filter=@skillhub/web RUN pnpm turbo run build --filter=@skillhub/web

View File

@@ -186,7 +186,7 @@ export default async function BrowsePage({ params, searchParams }: BrowsePagePro
const emptyStateTranslations = { const emptyStateTranslations = {
noResults: t('noResults') || 'No skills found', noResults: t('noResults') || 'No skills found',
noResultsWithQuery: t('noResultsWithQuery') || 'No results for "{query}"', noResultsWithQuery: t('noResultsWithQuery', { query: searchParamsResolved.q || '' }),
tryDifferent: t('emptyState.tryDifferent') || 'Try different search terms or adjust your filters', tryDifferent: t('emptyState.tryDifferent') || 'Try different search terms or adjust your filters',
clearFilters: t('emptyState.clearFilters') || 'Clear filters', clearFilters: t('emptyState.clearFilters') || 'Clear filters',
browseAll: t('emptyState.browseAll') || 'Browse Featured Skills', browseAll: t('emptyState.browseAll') || 'Browse Featured Skills',

View File

@@ -42,6 +42,7 @@ export default async function ClaimPage({
tabs: { tabs: {
remove: t('tabs.remove'), remove: t('tabs.remove'),
add: t('tabs.add'), add: t('tabs.add'),
removeRepo: t('tabs.removeRepo'),
}, },
form: { form: {
skillId: t('form.skillId'), skillId: t('form.skillId'),
@@ -78,6 +79,27 @@ export default async function ClaimPage({
foundSkillsIn: t('addSuccess.foundSkillsIn'), foundSkillsIn: t('addSuccess.foundSkillsIn'),
root: t('addSuccess.root'), root: t('addSuccess.root'),
andMore: t.raw('addSuccess.andMore') as string, andMore: t.raw('addSuccess.andMore') as string,
reEnabledTitle: t('addSuccess.reEnabledTitle'),
reEnabledDescription: t('addSuccess.reEnabledDescription'),
},
removeRepoForm: {
repoUrl: t('removeRepoForm.repoUrl'),
repoUrlPlaceholder: t('removeRepoForm.repoUrlPlaceholder'),
repoUrlHelp: t('removeRepoForm.repoUrlHelp'),
reason: t('removeRepoForm.reason'),
reasonPlaceholder: t('removeRepoForm.reasonPlaceholder'),
submit: t('removeRepoForm.submit'),
submitting: t('removeRepoForm.submitting'),
},
removeRepoSuccess: {
title: t('removeRepoSuccess.title'),
description: t.raw('removeRepoSuccess.description') as string,
descriptionZero: t('removeRepoSuccess.descriptionZero'),
},
removeRepoError: {
notOwner: t('removeRepoError.notOwner'),
invalidRepo: t('removeRepoError.invalidRepo'),
parseError: t('removeRepoError.parseError'),
}, },
error: { error: {
notOwner: t('error.notOwner'), notOwner: t('error.notOwner'),
@@ -90,6 +112,7 @@ export default async function ClaimPage({
rateLimitExceeded: t('error.rateLimitExceeded'), rateLimitExceeded: t('error.rateLimitExceeded'),
networkTimeout: t('error.networkTimeout'), networkTimeout: t('error.networkTimeout'),
generic: t('error.generic'), generic: t('error.generic'),
repoBlockedByOwner: t('error.repoBlockedByOwner'),
}, },
myRequests: { myRequests: {
title: t('myRequests.title'), title: t('myRequests.title'),

View File

@@ -10,6 +10,7 @@ import Link from 'next/link';
import type { Metadata } from 'next'; import type { Metadata } from 'next';
import { getPageAlternates } from '@/lib/seo'; import { getPageAlternates } from '@/lib/seo';
import { getOrSetCache, cacheKeys, cacheTTL } from '@/lib/cache'; import { getOrSetCache, cacheKeys, cacheTTL } from '@/lib/cache';
import { auth } from '@/lib/auth';
export const dynamic = 'force-dynamic'; export const dynamic = 'force-dynamic';
@@ -49,6 +50,8 @@ export default async function OwnerPage({ params, searchParams }: OwnerPageProps
const activeRepo = searchParamsResolved.repo || ''; const activeRepo = searchParamsResolved.repo || '';
const db = createDb(); const db = createDb();
const session = await auth();
const isOwner = session?.user?.username?.toLowerCase() === username.toLowerCase();
// Fetch stats and repo list with caching (30 min TTL), count is dynamic per filter // Fetch stats and repo list with caching (30 min TTL), count is dynamic per filter
const [stats, totalSkills, ownerRepos] = await Promise.all([ const [stats, totalSkills, ownerRepos] = await Promise.all([
@@ -256,16 +259,30 @@ export default async function OwnerPage({ params, searchParams }: OwnerPageProps
</div> </div>
</div> </div>
{/* Claim CTA - subtle inline hint */} {/* Claim CTA */}
<div className="flex items-center gap-2 mb-6 text-xs text-text-muted"> {isOwner ? (
<span>{t('claimCta')}</span> <div className="flex items-center gap-3 mb-6 px-4 py-3 bg-primary/5 border border-primary/20 rounded-lg">
<Link <div className="flex-1 text-sm text-text-secondary">
href={`/${locale}/claim`} {t('ownerManageCta')}
className="text-primary hover:underline font-medium whitespace-nowrap" </div>
> <Link
{t('claimButton')} href={`/${locale}/claim`}
</Link> className="text-sm text-primary hover:underline font-medium whitespace-nowrap"
</div> >
{t('ownerManageButton')}
</Link>
</div>
) : (
<div className="flex items-center gap-2 mb-6 text-xs text-text-muted">
<span>{t('claimCta')}</span>
<Link
href={`/${locale}/claim`}
className="text-primary hover:underline font-medium whitespace-nowrap"
>
{t('claimButton')}
</Link>
</div>
)}
{/* Repos + Skills */} {/* Repos + Skills */}
{repos.map((repo) => ( {repos.map((repo) => (

View File

@@ -21,7 +21,7 @@ async function getStats() {
const db = createDb(); const db = createDb();
// Browse-ready filter: exclude duplicates (matches browseReadyFilter in queries.ts) // Browse-ready filter: exclude duplicates (matches browseReadyFilter in queries.ts)
const browseReady = sql`${skills.isDuplicate} = false`; const browseReady = sql`${skills.isDuplicate} = false AND ${skills.isStale} = false`;
// Run all independent count queries in parallel // Run all independent count queries in parallel
const [skillsResult, downloadsResult, categories, contributorsResult, totalIndexedResult] = await Promise.all([ const [skillsResult, downloadsResult, categories, contributorsResult, totalIndexedResult] = await Promise.all([

View File

@@ -339,6 +339,33 @@ export default async function SkillPage({ params }: SkillPageProps) {
</div> </div>
)} )}
{/* Stale Skill Warning Banner */}
{dbSkill.isStale && (
<div className="bg-warning/10 border-b border-warning/20">
<div className="container-main py-3">
<div className="flex items-start gap-3 text-sm">
<span className="text-warning text-lg flex-shrink-0"></span>
<div>
<p className="text-warning-foreground dark:text-warning" dir="auto">
{isRTL
? 'این مهارت ممکن است از مخزن GitHub اصلی حذف یا جابجا شده باشد. فایل‌ها از حافظه پنهان SkillHub ارائه می‌شوند و ممکن است قدیمی باشند.'
: 'This skill may have been removed or moved from its GitHub repository. Files are served from the SkillHub cache and may be outdated.'}
</p>
<a
href={`https://github.com/${skill.author}/${skill.repo}`}
target="_blank"
rel="noopener noreferrer"
className="inline-flex items-center gap-1 mt-1 text-primary-600 hover:text-primary-700 dark:text-primary-400 dark:hover:text-primary-300 font-medium"
>
{isRTL ? 'بررسی مخزن GitHub' : 'Check GitHub repository'}
<ExternalLink className="w-3 h-3" />
</a>
</div>
</div>
</div>
</div>
)}
{/* Stats Bar */} {/* Stats Bar */}
<div className="bg-surface-elevated border-b border-border"> <div className="bg-surface-elevated border-b border-border">
<div className="container-main"> <div className="container-main">

View File

@@ -49,6 +49,9 @@ export async function GET(request: NextRequest) {
Math.max(parseInt(searchParams.get('owner_limit') ?? '0', 10) || 0, 0), Math.max(parseInt(searchParams.get('owner_limit') ?? '0', 10) || 0, 0),
10 10
); );
const currentReviewVersion = Math.max(
parseInt(searchParams.get('review_version') ?? '0', 10) || 0, 0
);
// Run counts in parallel // Run counts in parallel
const [totalPending, reReviews] = await Promise.all([ const [totalPending, reReviews] = await Promise.all([
@@ -68,6 +71,7 @@ export async function GET(request: NextRequest) {
securityPass, securityPass,
reReviewAll: true, reReviewAll: true,
ownerLimit, ownerLimit,
currentReviewVersion,
}) as typeof batch; }) as typeof batch;
batch = [...allBatch].slice(0, batchSize); batch = [...allBatch].slice(0, batchSize);
} else { } else {

View File

@@ -22,6 +22,8 @@ interface ReviewItem {
i18n_priority?: number; i18n_priority?: number;
content_hash_at_review?: string; content_hash_at_review?: string;
set_verified?: boolean; set_verified?: boolean;
review_version?: number;
reviewer?: string;
} }
function validateReviews(body: unknown): { reviews: ReviewItem[] } | { error: string } { function validateReviews(body: unknown): { reviews: ReviewItem[] } | { error: string } {
@@ -60,6 +62,18 @@ function validateReviews(body: unknown): { reviews: ReviewItem[] } | { error: st
return { error: `reviews[${i}].i18n_priority must be an integer 0-2` }; return { error: `reviews[${i}].i18n_priority must be an integer 0-2` };
} }
} }
// Validate review_version (positive integer)
if (item.review_version !== undefined && item.review_version !== null) {
if (typeof item.review_version !== 'number' || !Number.isInteger(item.review_version) || item.review_version < 1) {
return { error: `reviews[${i}].review_version must be a positive integer` };
}
}
// Validate reviewer (non-empty string, max 50 chars)
if (item.reviewer !== undefined && item.reviewer !== null) {
if (typeof item.reviewer !== 'string' || item.reviewer.length === 0 || item.reviewer.length > 50) {
return { error: `reviews[${i}].reviewer must be a non-empty string (max 50 chars)` };
}
}
} }
return { reviews: reviews as ReviewItem[] }; return { reviews: reviews as ReviewItem[] };
@@ -106,7 +120,7 @@ export async function POST(request: NextRequest) {
// Insert review rows // Insert review rows
const dbReviews = reviews.map((r) => ({ const dbReviews = reviews.map((r) => ({
skillId: r.skill_id, skillId: r.skill_id,
reviewer: 'claude-code' as const, reviewer: r.reviewer || 'claude-code',
aiScore: r.ai_score, aiScore: r.ai_score,
instructionQuality: r.instruction_quality, instructionQuality: r.instruction_quality,
descriptionPrecision: r.description_precision, descriptionPrecision: r.description_precision,
@@ -119,6 +133,7 @@ export async function POST(request: NextRequest) {
needsImprovement: r.needs_improvement ?? undefined, needsImprovement: r.needs_improvement ?? undefined,
i18nPriority: r.i18n_priority, i18nPriority: r.i18n_priority,
contentHashAtReview: r.content_hash_at_review, contentHashAtReview: r.content_hash_at_review,
reviewVersion: r.review_version,
})); }));
await skillReviewQueries.createBatch(db, dbReviews); await skillReviewQueries.createBatch(db, dbReviews);

View File

@@ -39,6 +39,7 @@ interface SkillFile {
content: string; content: string;
size: number; size: number;
isBinary: boolean; isBinary: boolean;
fetchFailed?: boolean;
} }
interface CachedFiles { interface CachedFiles {
@@ -136,7 +137,7 @@ export async function GET(request: NextRequest) {
} }
// Fetch skill folder contents from GitHub // Fetch skill folder contents from GitHub
const files = await fetchSkillFiles( const { files, fetchFailureCount } = await fetchSkillFiles(
githubOwner, githubOwner,
githubRepo, githubRepo,
skillPath, skillPath,
@@ -146,19 +147,22 @@ export async function GET(request: NextRequest) {
token token
); );
// === SAVE TO CACHE === // === SAVE TO CACHE (only if all files fetched successfully) ===
// Prepare cached files structure if (fetchFailureCount > 0) {
const filesToCache: CachedFiles = { console.warn(`[skill-files] Skipping cache for ${skillId}: ${fetchFailureCount} file(s) failed to fetch`);
fetchedAt: new Date().toISOString(), } else {
commitSha: commitSha || 'unknown', const filesToCache: CachedFiles = {
totalSize: files.reduce((sum, f) => sum + f.size, 0), fetchedAt: new Date().toISOString(),
items: files, commitSha: commitSha || 'unknown',
}; totalSize: files.reduce((sum, f) => sum + f.size, 0),
items: files.map(({ fetchFailed: _, ...rest }) => rest),
};
// Save to database (async, don't block response) // Save to database (async, don't block response)
skillQueries.updateCachedFiles(db, skillId, filesToCache).catch((err) => { skillQueries.updateCachedFiles(db, skillId, filesToCache).catch((err) => {
console.error(`[skill-files] Failed to cache files for ${skillId}:`, err); console.error(`[skill-files] Failed to cache files for ${skillId}:`, err);
}); });
}
// Note: Download count is NOT incremented here. // Note: Download count is NOT incremented here.
// It should only be incremented in /api/skills/install after successful download. // It should only be incremented in /api/skills/install after successful download.
@@ -180,6 +184,7 @@ export async function GET(request: NextRequest) {
downloadUrl: item.isBinary ? `https://raw.githubusercontent.com/${githubOwner}/${githubRepo}/${branch || 'main'}/${skillPath}/${item.path}` : undefined, downloadUrl: item.isBinary ? `https://raw.githubusercontent.com/${githubOwner}/${githubRepo}/${branch || 'main'}/${skillPath}/${item.path}` : undefined,
})), })),
fromCache: false, fromCache: false,
...(fetchFailureCount > 0 ? { fetchFailures: fetchFailureCount } : {}),
}, { }, {
headers: createRateLimitHeaders(rateLimitResult), headers: createRateLimitHeaders(rateLimitResult),
}); });
@@ -204,6 +209,44 @@ export async function GET(request: NextRequest) {
} }
if (errorMessage.includes('404')) { if (errorMessage.includes('404')) {
// Increment stale check (non-blocking) — after 3 consecutive 404s, skill is marked stale
const skillId404 = request.nextUrl.searchParams.get('id');
if (skillId404) {
skillQueries.incrementStaleCheck(db, skillId404).catch((err) => {
console.error(`[skill-files] Failed to increment stale check for ${skillId404}:`, err);
});
}
// If we have cached files in DB, serve them with stale warning instead of 404
if (skillId404) {
const skill404 = await skillQueries.getById(db, skillId404);
if (skill404?.cachedFiles) {
const cached = skill404.cachedFiles as CachedFiles;
return NextResponse.json({
skillId: skillId404,
githubOwner: skill404.githubOwner,
githubRepo: skill404.githubRepo,
skillPath: skill404.skillPath,
branch: skill404.branch || 'main',
sourceFormat: skill404.sourceFormat || 'skill.md',
files: cached.items.map(item => ({
name: item.name,
path: item.path,
type: 'file' as const,
size: item.size,
content: item.isBinary ? undefined : item.content,
downloadUrl: item.isBinary ? `https://raw.githubusercontent.com/${skill404.githubOwner}/${skill404.githubRepo}/${skill404.branch || 'main'}/${skill404.skillPath}/${item.path}` : undefined,
})),
fromCache: true,
cachedAt: cached.fetchedAt,
isStale: true,
staleWarning: 'This skill may have been removed from GitHub. Files served from cache.',
}, {
headers: createRateLimitHeaders(rateLimitResult),
});
}
}
return NextResponse.json( return NextResponse.json(
{ error: 'Skill files not found on GitHub', code: 'GITHUB_NOT_FOUND' }, { error: 'Skill files not found on GitHub', code: 'GITHUB_NOT_FOUND' },
{ status: 404 } { status: 404 }
@@ -314,7 +357,7 @@ async function fetchSkillFiles(
depth: number = 0, depth: number = 0,
headers: Record<string, string>, headers: Record<string, string>,
token: string | null token: string | null
): Promise<SkillFile[]> { ): Promise<{ files: SkillFile[]; fetchFailureCount: number }> {
// Phase 1: Collect all file entries from directory tree // Phase 1: Collect all file entries from directory tree
const entries = await collectFileEntries(owner, repo, path, ref, depth, headers, token); const entries = await collectFileEntries(owner, repo, path, ref, depth, headers, token);
@@ -326,6 +369,7 @@ async function fetchSkillFiles(
// Phase 2: Fetch file contents in parallel // Phase 2: Fetch file contents in parallel
let totalSize = 0; let totalSize = 0;
let fetchFailureCount = 0;
const tasks = entries.map((entry) => async (): Promise<SkillFile> => { const tasks = entries.map((entry) => async (): Promise<SkillFile> => {
const { item, relativePath } = entry; const { item, relativePath } = entry;
const isBinary = !isTextFile(item.name); const isBinary = !isTextFile(item.name);
@@ -340,15 +384,18 @@ async function fetchSkillFiles(
content, content,
size: item.size, size: item.size,
isBinary: false, isBinary: false,
fetchFailed: false,
}; };
} catch { } catch {
// Content fetch failed — return as binary with download URL // Content fetch failed — mark as failed so we skip caching
fetchFailureCount++;
return { return {
name: item.name, name: item.name,
path: relativePath, path: relativePath,
content: '', content: '',
size: item.size, size: item.size,
isBinary: true, isBinary: true,
fetchFailed: true,
}; };
} }
} }
@@ -360,10 +407,12 @@ async function fetchSkillFiles(
content: '', content: '',
size: item.size, size: item.size,
isBinary: true, isBinary: true,
fetchFailed: false,
}; };
}); });
return parallelLimit(tasks, PARALLEL_FETCH_LIMIT); const files = await parallelLimit(tasks, PARALLEL_FETCH_LIMIT);
return { files, fetchFailureCount };
} }
/** /**

View File

@@ -0,0 +1,229 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { NextRequest } from 'next/server';
// Mock auth
const mockAuth = vi.fn();
vi.mock('@/lib/auth', () => ({
auth: () => mockAuth(),
}));
// Mock sanitize
vi.mock('@/lib/sanitize', () => ({
sanitizeReason: (r: string) => r,
}));
// Mock email
vi.mock('@/lib/email', () => ({
sendClaimSubmittedEmail: vi.fn().mockResolvedValue(undefined),
}));
// Mock db
vi.mock('@skillhub/db', () => ({
createDb: vi.fn(() => ({})),
userQueries: {
getByGithubId: vi.fn(),
upsertFromGithub: vi.fn(),
},
skillQueries: {
unblockByRepo: vi.fn(),
},
addRequestQueries: {
hasPendingRequest: vi.fn(),
create: vi.fn(),
updateStatus: vi.fn(),
},
discoveredRepoQueries: {
getById: vi.fn(),
upsert: vi.fn(),
unblockRepo: vi.fn(),
},
}));
import { POST, GET } from './route';
import { userQueries, skillQueries, addRequestQueries, discoveredRepoQueries } from '@skillhub/db';
function createMockUser(overrides: Partial<{ id: string; githubId: string; email: string }> = {}) {
return {
id: 'user-123',
githubId: 'gh-12345',
username: 'testuser',
email: 'test@example.com',
preferredLocale: 'en',
...overrides,
};
}
function createRequest(body: unknown) {
return new NextRequest('http://localhost:3000/api/skills/add-request', {
method: 'POST',
body: JSON.stringify(body),
headers: { 'Content-Type': 'application/json' },
});
}
describe('/api/skills/add-request', () => {
beforeEach(() => {
vi.clearAllMocks();
// Disable real fetch by default
vi.stubGlobal('fetch', vi.fn());
});
describe('GET', () => {
it('should return 401 when not authenticated', async () => {
mockAuth.mockResolvedValue(null);
const response = await GET();
const data = await response.json();
expect(response.status).toBe(401);
expect(data.error).toBeDefined();
});
it('should return empty requests for unknown user', async () => {
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345' } });
vi.mocked(userQueries.getByGithubId).mockResolvedValue(null as any);
const response = await GET();
const data = await response.json();
expect(response.status).toBe(200);
expect(data.requests).toEqual([]);
});
});
describe('POST', () => {
it('should return 401 when not authenticated', async () => {
mockAuth.mockResolvedValue(null);
const response = await POST(createRequest({ repositoryUrl: 'https://github.com/owner/repo' }));
const data = await response.json();
expect(response.status).toBe(401);
expect(data.code).toBe('AUTH_REQUIRED');
});
it('should return 400 for missing repositoryUrl', async () => {
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345', username: 'testuser' } });
const response = await POST(createRequest({}));
const data = await response.json();
expect(response.status).toBe(400);
expect(data.code).toBe('INVALID_INPUT');
});
it('should return 400 for invalid GitHub URL', async () => {
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345', username: 'testuser' } });
const mockUser = createMockUser();
vi.mocked(userQueries.getByGithubId).mockResolvedValue(mockUser as any);
// gitlab.com does not contain the substring 'github.com'
const response = await POST(createRequest({ repositoryUrl: 'https://gitlab.com/owner/repo' }));
const data = await response.json();
expect(response.status).toBe(400);
expect(data.code).toBe('INVALID_URL');
});
it('should return 403 REPO_BLOCKED_BY_OWNER when non-owner tries to add blocked repo', async () => {
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345', username: 'non-owner' } });
const mockUser = createMockUser({ githubId: 'gh-12345' });
vi.mocked(userQueries.getByGithubId).mockResolvedValue(mockUser as any);
vi.mocked(discoveredRepoQueries.getById).mockResolvedValue({
id: 'rawveg/skillsforge-marketplace',
isBlocked: true,
} as any);
// GitHub API says the owner is 'rawveg', not 'non-owner'
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ owner: { login: 'rawveg' }, default_branch: 'main', private: false }),
} as any));
const response = await POST(createRequest({ repositoryUrl: 'https://github.com/rawveg/skillsforge-marketplace' }));
const data = await response.json();
expect(response.status).toBe(403);
expect(data.code).toBe('REPO_BLOCKED_BY_OWNER');
});
it('should re-enable repo when owner re-adds their blocked repo', async () => {
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345', username: 'rawveg' } });
const mockUser = createMockUser({ githubId: 'gh-12345' });
vi.mocked(userQueries.getByGithubId).mockResolvedValue(mockUser as any);
vi.mocked(discoveredRepoQueries.getById).mockResolvedValue({
id: 'rawveg/skillsforge-marketplace',
isBlocked: true,
} as any);
vi.mocked(skillQueries.unblockByRepo).mockResolvedValue(undefined);
vi.mocked(discoveredRepoQueries.unblockRepo).mockResolvedValue(undefined);
vi.mocked(addRequestQueries.hasPendingRequest).mockResolvedValue(false as any);
vi.mocked(addRequestQueries.create).mockResolvedValue('req-123' as any);
vi.mocked(addRequestQueries.updateStatus).mockResolvedValue(undefined);
vi.mocked(discoveredRepoQueries.upsert).mockResolvedValue(undefined as any);
// First fetch: ownership check (inside blocked-repo logic)
// Second fetch: repo validation (validateGitHubRepo → repoResponse)
// Third fetch: tree scan (findSkillMdFiles)
vi.stubGlobal('fetch', vi.fn()
.mockResolvedValueOnce({
ok: true,
json: async () => ({ owner: { login: 'rawveg' }, default_branch: 'main', private: false }),
} as any)
.mockResolvedValueOnce({
ok: true,
json: async () => ({ owner: { login: 'rawveg' }, default_branch: 'main', private: false }),
} as any)
.mockResolvedValueOnce({
ok: true,
json: async () => ({
tree: [
{ path: 'SKILL.md', type: 'blob' },
],
truncated: false,
}),
} as any)
);
const response = await POST(createRequest({ repositoryUrl: 'https://github.com/rawveg/skillsforge-marketplace' }));
const data = await response.json();
expect(response.status).toBe(200);
expect(data.success).toBe(true);
expect(data.reEnabled).toBe(true);
expect(skillQueries.unblockByRepo).toHaveBeenCalledWith(expect.anything(), 'rawveg', 'skillsforge-marketplace');
expect(discoveredRepoQueries.unblockRepo).toHaveBeenCalledWith(expect.anything(), 'rawveg/skillsforge-marketplace');
});
it('should proceed normally when repo is not blocked (Case C)', async () => {
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345', username: 'newuser' } });
const mockUser = createMockUser();
vi.mocked(userQueries.getByGithubId).mockResolvedValue(mockUser as any);
vi.mocked(discoveredRepoQueries.getById).mockResolvedValue(null as any);
vi.mocked(addRequestQueries.hasPendingRequest).mockResolvedValue(false as any);
vi.mocked(addRequestQueries.create).mockResolvedValue('req-456' as any);
vi.mocked(addRequestQueries.updateStatus).mockResolvedValue(undefined);
vi.mocked(discoveredRepoQueries.upsert).mockResolvedValue(undefined as any);
vi.stubGlobal('fetch', vi.fn()
.mockResolvedValueOnce({
ok: true,
json: async () => ({ owner: { login: 'someowner' }, default_branch: 'main', private: false }),
} as any)
.mockResolvedValueOnce({
ok: true,
json: async () => ({
tree: [{ path: 'SKILL.md', type: 'blob' }],
truncated: false,
}),
} as any)
);
const response = await POST(createRequest({ repositoryUrl: 'https://github.com/someowner/newrepo' }));
const data = await response.json();
expect(response.status).toBe(200);
expect(data.success).toBe(true);
expect(data.reEnabled).toBeUndefined();
});
});
});

View File

@@ -1,7 +1,7 @@
import type { NextRequest } from 'next/server'; import type { NextRequest } from 'next/server';
import { NextResponse } from 'next/server'; import { NextResponse } from 'next/server';
import { auth } from '@/lib/auth'; import { auth } from '@/lib/auth';
import { createDb, addRequestQueries, discoveredRepoQueries, userQueries } from '@skillhub/db'; import { createDb, addRequestQueries, discoveredRepoQueries, skillQueries, userQueries } from '@skillhub/db';
import { sanitizeReason } from '@/lib/sanitize'; import { sanitizeReason } from '@/lib/sanitize';
import { sendClaimSubmittedEmail } from '@/lib/email'; import { sendClaimSubmittedEmail } from '@/lib/email';
@@ -337,6 +337,52 @@ export async function POST(request: NextRequest) {
); );
} }
// Check if repo was previously blocked by its owner
let reEnabled = false;
const existingRepo = await discoveredRepoQueries.getById(db, `${parsed.owner}/${parsed.repo}`);
if (existingRepo?.isBlocked) {
// Verify whether the current requester is the repo owner
let requesterIsOwner = false;
try {
const ownerCheckRes = await fetch(
`https://api.github.com/repos/${parsed.owner}/${parsed.repo}`,
{
headers: {
Accept: 'application/vnd.github.v3+json',
'User-Agent': 'SkillHub',
...(process.env.GITHUB_TOKEN && {
Authorization: `Bearer ${process.env.GITHUB_TOKEN}`,
}),
},
signal: AbortSignal.timeout(10000),
}
);
if (ownerCheckRes.ok) {
const repoData = await ownerCheckRes.json() as { owner?: { login?: string } };
requesterIsOwner =
repoData.owner?.login?.toLowerCase() === session.user.username.toLowerCase();
}
} catch {
// If we can't reach GitHub, treat as non-owner (safe default)
}
if (!requesterIsOwner) {
// Case B: repo blocked by its owner, non-owner trying to re-add
return NextResponse.json(
{
error: 'This repository was removed by its owner and cannot be re-added.',
code: 'REPO_BLOCKED_BY_OWNER',
},
{ status: 403 }
);
}
// Case A: owner is re-enabling their previously blocked repo
await skillQueries.unblockByRepo(db, parsed.owner, parsed.repo);
await discoveredRepoQueries.unblockRepo(db, `${parsed.owner}/${parsed.repo}`);
reEnabled = true;
}
// Check if user already has a pending request for this repository + path combination // Check if user already has a pending request for this repository + path combination
const hasPending = await addRequestQueries.hasPendingRequest( const hasPending = await addRequestQueries.hasPendingRequest(
db, db,
@@ -438,6 +484,7 @@ export async function POST(request: NextRequest) {
skillCount: validation.skillPaths.length, skillCount: validation.skillPaths.length,
skillPaths: validation.skillPaths, skillPaths: validation.skillPaths,
message, message,
...(reEnabled && { reEnabled: true }),
}); });
} catch (error) { } catch (error) {
console.error('Error creating add request:', error); console.error('Error creating add request:', error);

View File

@@ -0,0 +1,163 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { NextRequest } from 'next/server';
// Mock auth
const mockAuth = vi.fn();
vi.mock('@/lib/auth', () => ({
auth: () => mockAuth(),
}));
// Mock sanitize
vi.mock('@/lib/sanitize', () => ({
sanitizeReason: (r: string) => r,
}));
// Mock db
vi.mock('@skillhub/db', () => ({
createDb: vi.fn(() => ({})),
userQueries: {
getByGithubId: vi.fn(),
upsertFromGithub: vi.fn(),
},
skillQueries: {
countByRepo: vi.fn(),
blockByRepo: vi.fn(),
},
discoveredRepoQueries: {
blockRepo: vi.fn(),
},
}));
import { POST } from './route';
import { userQueries, skillQueries, discoveredRepoQueries } from '@skillhub/db';
function createMockUser(overrides: Partial<{ id: string; githubId: string }> = {}) {
return {
id: 'user-123',
githubId: 'gh-12345',
username: 'rawveg',
...overrides,
};
}
function createRequest(body: unknown) {
return new NextRequest('http://localhost:3000/api/skills/repo-removal-request', {
method: 'POST',
body: JSON.stringify(body),
headers: { 'Content-Type': 'application/json' },
});
}
describe('/api/skills/repo-removal-request', () => {
beforeEach(() => {
vi.clearAllMocks();
vi.stubGlobal('fetch', vi.fn());
});
describe('POST', () => {
it('should return 401 when not authenticated', async () => {
mockAuth.mockResolvedValue(null);
const response = await POST(createRequest({ repoUrl: 'rawveg/skillsforge-marketplace' }));
const data = await response.json();
expect(response.status).toBe(401);
expect(data.code).toBe('AUTH_REQUIRED');
});
it('should return 400 for missing repoUrl', async () => {
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345', username: 'rawveg' } });
const response = await POST(createRequest({}));
expect(response.status).toBe(400);
});
it('should return 400 PARSE_ERROR for invalid repo format', async () => {
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345', username: 'rawveg' } });
const mockUser = createMockUser();
vi.mocked(userQueries.getByGithubId).mockResolvedValue(mockUser as any);
const response = await POST(createRequest({ repoUrl: 'not-a-valid-format' }));
const data = await response.json();
expect(response.status).toBe(400);
expect(data.code).toBe('PARSE_ERROR');
});
it('should return 404 INVALID_REPO when repo not found on GitHub', async () => {
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345', username: 'rawveg' } });
const mockUser = createMockUser();
vi.mocked(userQueries.getByGithubId).mockResolvedValue(mockUser as any);
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
ok: false,
status: 404,
} as any));
const response = await POST(createRequest({ repoUrl: 'rawveg/skillsforge-marketplace' }));
const data = await response.json();
expect(response.status).toBe(404);
expect(data.code).toBe('INVALID_REPO');
});
it('should return 403 NOT_OWNER when requester is not repo owner', async () => {
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345', username: 'other-user' } });
const mockUser = createMockUser({ githubId: 'gh-12345' });
vi.mocked(userQueries.getByGithubId).mockResolvedValue(mockUser as any);
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ owner: { login: 'rawveg' } }),
} as any));
const response = await POST(createRequest({ repoUrl: 'rawveg/skillsforge-marketplace' }));
const data = await response.json();
expect(response.status).toBe(403);
expect(data.code).toBe('NOT_OWNER');
});
it('should block all skills and return success when owner removes repo', async () => {
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345', username: 'rawveg' } });
const mockUser = createMockUser();
vi.mocked(userQueries.getByGithubId).mockResolvedValue(mockUser as any);
vi.mocked(skillQueries.countByRepo).mockResolvedValue(16 as any);
vi.mocked(skillQueries.blockByRepo).mockResolvedValue(undefined);
vi.mocked(discoveredRepoQueries.blockRepo).mockResolvedValue(undefined);
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ owner: { login: 'rawveg' } }),
} as any));
const response = await POST(createRequest({ repoUrl: 'rawveg/skillsforge-marketplace', reason: 'I want my skills removed' }));
const data = await response.json();
expect(response.status).toBe(200);
expect(data.success).toBe(true);
expect(data.blockedCount).toBe(16);
expect(data.repo).toBe('rawveg/skillsforge-marketplace');
expect(skillQueries.blockByRepo).toHaveBeenCalledWith(expect.anything(), 'rawveg', 'skillsforge-marketplace');
expect(discoveredRepoQueries.blockRepo).toHaveBeenCalledWith(expect.anything(), 'rawveg/skillsforge-marketplace');
});
it('should accept full GitHub URL format', async () => {
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345', username: 'rawveg' } });
const mockUser = createMockUser();
vi.mocked(userQueries.getByGithubId).mockResolvedValue(mockUser as any);
vi.mocked(skillQueries.countByRepo).mockResolvedValue(0 as any);
vi.mocked(skillQueries.blockByRepo).mockResolvedValue(undefined);
vi.mocked(discoveredRepoQueries.blockRepo).mockResolvedValue(undefined);
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ owner: { login: 'rawveg' } }),
} as any));
const response = await POST(createRequest({ repoUrl: 'https://github.com/rawveg/skillsforge-marketplace' }));
const data = await response.json();
expect(response.status).toBe(200);
expect(data.success).toBe(true);
expect(data.repo).toBe('rawveg/skillsforge-marketplace');
});
});
});

View File

@@ -0,0 +1,158 @@
import type { NextRequest } from 'next/server';
import { NextResponse } from 'next/server';
import { auth } from '@/lib/auth';
import { createDb, skillQueries, discoveredRepoQueries, userQueries } from '@skillhub/db';
import { sanitizeReason } from '@/lib/sanitize';
export const dynamic = 'force-dynamic';
const db = createDb();
function parseOwnerRepo(input: string): { owner: string; repo: string } | null {
try {
if (input.startsWith('https://') || input.startsWith('http://')) {
const url = new URL(input);
if (!url.hostname.includes('github.com')) return null;
const parts = url.pathname.split('/').filter(Boolean);
if (parts.length < 2) return null;
return { owner: parts[0], repo: parts[1] };
}
const parts = input.split('/').filter(Boolean);
if (parts.length < 2) return null;
return { owner: parts[0], repo: parts[1] };
} catch {
return null;
}
}
/**
* POST /api/skills/repo-removal-request
* Block all skills from a GitHub repository at once.
* Verifies the authenticated user owns the repo, then blocks all skills and
* prevents the repo from being re-indexed.
*/
export async function POST(request: NextRequest) {
try {
const session = await auth();
if (!session?.user?.githubId || !session.user.username) {
return NextResponse.json(
{ error: 'Authentication required', code: 'AUTH_REQUIRED' },
{ status: 401 }
);
}
const body = await request.json();
const { repoUrl, reason } = body;
if (!repoUrl) {
return NextResponse.json(
{ error: 'repoUrl is required', code: 'INVALID_INPUT' },
{ status: 400 }
);
}
const parsed = parseOwnerRepo(String(repoUrl).trim());
if (!parsed) {
return NextResponse.json(
{ error: 'Invalid format. Use https://github.com/owner/repo or owner/repo', code: 'PARSE_ERROR' },
{ status: 400 }
);
}
const { owner, repo } = parsed;
const username = session.user.username;
// Get or create user
let dbUser = await userQueries.getByGithubId(db, session.user.githubId);
if (!dbUser) {
dbUser = await userQueries.upsertFromGithub(db, {
githubId: session.user.githubId,
username: session.user.username,
displayName: session.user.name || undefined,
email: session.user.email || undefined,
avatarUrl: session.user.image || undefined,
});
}
if (!dbUser) {
return NextResponse.json(
{ error: 'Failed to create user record', code: 'USER_CREATE_FAILED' },
{ status: 500 }
);
}
// Verify repo ownership via GitHub API
let isOwner = false;
let githubError: string | null = null;
try {
const repoResponse = await fetch(
`https://api.github.com/repos/${owner}/${repo}`,
{
headers: {
Accept: 'application/vnd.github.v3+json',
'User-Agent': 'SkillHub',
},
signal: AbortSignal.timeout(10000),
}
);
if (repoResponse.ok) {
const repoData = await repoResponse.json();
if (repoData.owner?.login?.toLowerCase() === username.toLowerCase()) {
isOwner = true;
}
} else if (repoResponse.status === 404) {
return NextResponse.json(
{ error: 'Repository not found on GitHub', code: 'INVALID_REPO' },
{ status: 404 }
);
} else if (repoResponse.status === 403) {
githubError = 'GitHub API rate limit exceeded. Please try again later.';
}
} catch (fetchError) {
console.error('GitHub API fetch error:', fetchError);
githubError = 'Failed to verify repository ownership';
}
if (githubError) {
return NextResponse.json(
{ error: githubError, code: 'GITHUB_ERROR' },
{ status: 502 }
);
}
if (!isOwner) {
return NextResponse.json(
{ error: 'You are not the owner of this repository', code: 'NOT_OWNER' },
{ status: 403 }
);
}
// Count skills before blocking (for response message)
const blockedCount = await skillQueries.countByRepo(db, owner, repo);
// Block all skills from the repo
await skillQueries.blockByRepo(db, owner, repo);
// Block the discovered_repo entry to prevent re-indexing
await discoveredRepoQueries.blockRepo(db, `${owner}/${repo}`);
const sanitizedReason = reason ? sanitizeReason(String(reason)) : undefined;
console.log(
`[RepoRemoval] ${username} removed ${owner}/${repo} (${blockedCount} skills blocked)${sanitizedReason ? `${sanitizedReason}` : ''}`
);
return NextResponse.json({
success: true,
blockedCount,
repo: `${owner}/${repo}`,
});
} catch (error) {
console.error('Error processing repo removal request:', error);
return NextResponse.json(
{ error: 'Internal server error', code: 'SERVER_ERROR' },
{ status: 500 }
);
}
}

View File

@@ -35,7 +35,7 @@ export async function GET(request: NextRequest) {
} }
// Browse-ready filter: exclude duplicates (matches browseReadyFilter in queries.ts) // Browse-ready filter: exclude duplicates (matches browseReadyFilter in queries.ts)
const browseReady = sql`${skills.isDuplicate} = false`; const browseReady = sql`${skills.isDuplicate} = false AND ${skills.isStale} = false`;
// Browse-ready skill stats (skills count + contributors) // Browse-ready skill stats (skills count + contributors)
const statsResult = await db const statsResult = await db

View File

@@ -470,7 +470,7 @@ export function EmptyState({ query, hasFilters, locale = 'en', translations }: E
<Sparkles className="w-10 h-10 text-text-muted" /> <Sparkles className="w-10 h-10 text-text-muted" />
</div> </div>
<h3 className="text-xl font-semibold text-text-primary mb-2"> <h3 className="text-xl font-semibold text-text-primary mb-2">
{query ? translations.noResultsWithQuery.replace('{query}', query) : translations.noResults} {query ? translations.noResultsWithQuery : translations.noResults}
</h3> </h3>
<p className="text-text-secondary mb-6 max-w-md mx-auto"> <p className="text-text-secondary mb-6 max-w-md mx-auto">
{translations.tryDifferent} {translations.tryDifferent}

View File

@@ -2,7 +2,7 @@
import { useState, useEffect, useCallback } from 'react'; import { useState, useEffect, useCallback } from 'react';
import { useSession, signIn } from 'next-auth/react'; import { useSession, signIn } from 'next-auth/react';
import { Loader2, CheckCircle, AlertCircle, Github, Clock, Plus, Minus, ChevronDown, ChevronUp, ExternalLink } from 'lucide-react'; import { Loader2, CheckCircle, AlertCircle, Github, Clock, Plus, Minus, ChevronDown, ChevronUp, ExternalLink, Trash2 } from 'lucide-react';
import { fetchWithCsrf } from '@/lib/csrf-client'; import { fetchWithCsrf } from '@/lib/csrf-client';
const isMirror = process.env.NEXT_PUBLIC_IS_PRIMARY === 'false'; const isMirror = process.env.NEXT_PUBLIC_IS_PRIMARY === 'false';
@@ -33,6 +33,26 @@ interface ClaimFormProps {
tabs: { tabs: {
remove: string; remove: string;
add: string; add: string;
removeRepo: string;
};
removeRepoForm: {
repoUrl: string;
repoUrlPlaceholder: string;
repoUrlHelp: string;
reason: string;
reasonPlaceholder: string;
submit: string;
submitting: string;
};
removeRepoSuccess: {
title: string;
description: string;
descriptionZero: string;
};
removeRepoError: {
notOwner: string;
invalidRepo: string;
parseError: string;
}; };
form: { form: {
skillId: string; skillId: string;
@@ -69,6 +89,8 @@ interface ClaimFormProps {
foundSkillsIn: string; foundSkillsIn: string;
root: string; root: string;
andMore: string; andMore: string;
reEnabledTitle: string;
reEnabledDescription: string;
}; };
error: { error: {
notOwner: string; notOwner: string;
@@ -81,6 +103,7 @@ interface ClaimFormProps {
rateLimitExceeded: string; rateLimitExceeded: string;
networkTimeout: string; networkTimeout: string;
generic: string; generic: string;
repoBlockedByOwner: string;
}; };
myRequests: { myRequests: {
title: string; title: string;
@@ -122,15 +145,15 @@ interface AddRequest {
export function ClaimForm({ translations }: ClaimFormProps) { export function ClaimForm({ translations }: ClaimFormProps) {
const { data: session, status } = useSession(); const { data: session, status } = useSession();
const [activeTab, setActiveTab] = useState<'remove' | 'add'>('add'); // Default to 'add' tab const [activeTab, setActiveTab] = useState<'remove' | 'add' | 'remove-repo'>('add'); // Default to 'add' tab
const [expandedRequests, setExpandedRequests] = useState<Set<string>>(new Set()); const [expandedRequests, setExpandedRequests] = useState<Set<string>>(new Set());
// Read tab from URL hash on mount and update on hash change // Read tab from URL hash on mount and update on hash change
useEffect(() => { useEffect(() => {
const readHashTab = () => { const readHashTab = () => {
const hash = window.location.hash.slice(1); const hash = window.location.hash.slice(1);
if (hash === 'remove' || hash === 'add') { if (hash === 'remove' || hash === 'add' || hash === 'remove-repo') {
setActiveTab(hash); setActiveTab(hash as 'remove' | 'add' | 'remove-repo');
} }
}; };
readHashTab(); readHashTab();
@@ -139,7 +162,7 @@ export function ClaimForm({ translations }: ClaimFormProps) {
}, []); }, []);
// Update URL hash when tab changes // Update URL hash when tab changes
const handleTabChange = useCallback((tab: 'remove' | 'add') => { const handleTabChange = useCallback((tab: 'remove' | 'add' | 'remove-repo') => {
setActiveTab(tab); setActiveTab(tab);
window.history.replaceState(null, '', `#${tab}`); window.history.replaceState(null, '', `#${tab}`);
}, []); }, []);
@@ -177,9 +200,19 @@ export function ClaimForm({ translations }: ClaimFormProps) {
const [addHasSkillMd, setAddHasSkillMd] = useState(true); const [addHasSkillMd, setAddHasSkillMd] = useState(true);
const [addSkillCount, setAddSkillCount] = useState(0); const [addSkillCount, setAddSkillCount] = useState(0);
const [addSkillPaths, setAddSkillPaths] = useState<string[]>([]); const [addSkillPaths, setAddSkillPaths] = useState<string[]>([]);
const [addReEnabled, setAddReEnabled] = useState(false);
const [addRequests, setAddRequests] = useState<AddRequest[]>([]); const [addRequests, setAddRequests] = useState<AddRequest[]>([]);
const [loadingAddRequests, setLoadingAddRequests] = useState(false); const [loadingAddRequests, setLoadingAddRequests] = useState(false);
// Remove-repo form state
const [repoUrl, setRepoUrl] = useState('');
const [repoReason, setRepoReason] = useState('');
const [isSubmittingRepo, setIsSubmittingRepo] = useState(false);
const [repoError, setRepoError] = useState('');
const [repoSuccess, setRepoSuccess] = useState(false);
const [repoBlockedCount, setRepoBlockedCount] = useState(0);
const [repoName, setRepoName] = useState('');
// Fetch user's existing requests // Fetch user's existing requests
useEffect(() => { useEffect(() => {
if (session) { if (session) {
@@ -331,6 +364,9 @@ export function ClaimForm({ translations }: ClaimFormProps) {
case 'ALREADY_PENDING': case 'ALREADY_PENDING':
setAddError(translations.error.alreadyPending); setAddError(translations.error.alreadyPending);
break; break;
case 'REPO_BLOCKED_BY_OWNER':
setAddError(translations.error.repoBlockedByOwner);
break;
case 'AUTH_REQUIRED': case 'AUTH_REQUIRED':
signIn('github'); signIn('github');
return; return;
@@ -342,6 +378,7 @@ export function ClaimForm({ translations }: ClaimFormProps) {
} }
setAddSuccess(true); setAddSuccess(true);
setAddReEnabled(data.reEnabled ?? false);
setAddHasSkillMd(data.hasSkillMd ?? true); setAddHasSkillMd(data.hasSkillMd ?? true);
setAddSkillCount(data.skillCount ?? 0); setAddSkillCount(data.skillCount ?? 0);
setAddSkillPaths(data.skillPaths ?? []); setAddSkillPaths(data.skillPaths ?? []);
@@ -355,6 +392,61 @@ export function ClaimForm({ translations }: ClaimFormProps) {
} }
}; };
const handleRepoRemovalSubmit = async (e: React.FormEvent) => {
e.preventDefault();
if (!session) {
signIn('github');
return;
}
if (!repoUrl.trim()) return;
setIsSubmittingRepo(true);
setRepoError('');
setRepoSuccess(false);
try {
const res = await fetchWithCsrf('/api/skills/repo-removal-request', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ repoUrl: repoUrl.trim(), reason: repoReason.trim() }),
});
const data = await res.json();
if (!res.ok) {
switch (data.code) {
case 'NOT_OWNER':
setRepoError(translations.removeRepoError.notOwner);
break;
case 'INVALID_REPO':
setRepoError(translations.removeRepoError.invalidRepo);
break;
case 'PARSE_ERROR':
setRepoError(translations.removeRepoError.parseError);
break;
case 'AUTH_REQUIRED':
signIn('github');
return;
default:
console.error('Repo removal error:', data);
setRepoError(translations.error.generic);
}
return;
}
setRepoSuccess(true);
setRepoBlockedCount(data.blockedCount ?? 0);
setRepoName(data.repo ?? repoUrl.trim());
setRepoUrl('');
setRepoReason('');
} catch {
setRepoError(translations.error.generic);
} finally {
setIsSubmittingRepo(false);
}
};
const getStatusBadge = (requestStatus: string) => { const getStatusBadge = (requestStatus: string) => {
switch (requestStatus) { switch (requestStatus) {
case 'pending': case 'pending':
@@ -457,11 +549,40 @@ export function ClaimForm({ translations }: ClaimFormProps) {
); );
} }
// Repo removal success state
if (repoSuccess) {
return (
<div className="card p-8 text-center">
<CheckCircle className="w-12 h-12 mx-auto mb-4 text-success" />
<h2 className="text-xl font-semibold text-text-primary mb-2">
{translations.removeRepoSuccess.title}
</h2>
<p className="text-text-secondary mb-6">
{repoBlockedCount > 0
? translations.removeRepoSuccess.description
.replace('{count}', String(repoBlockedCount))
.replace('{repo}', repoName)
: translations.removeRepoSuccess.descriptionZero}
</p>
<button
onClick={() => { setRepoSuccess(false); setRepoBlockedCount(0); setRepoName(''); }}
className="btn-secondary"
>
{translations.success.viewRequests}
</button>
</div>
);
}
// Add success state // Add success state
if (addSuccess) { if (addSuccess) {
// Determine which message to show // Determine which message to show
let successTitle = translations.addSuccess.title;
let successDescription: string | React.ReactNode; let successDescription: string | React.ReactNode;
if (addSkillCount > 1) { if (addReEnabled) {
successTitle = translations.addSuccess.reEnabledTitle;
successDescription = translations.addSuccess.reEnabledDescription;
} else if (addSkillCount > 1) {
successDescription = ( successDescription = (
<> <>
{translations.addSuccess.descriptionMultiplePrefix} {translations.addSuccess.descriptionMultiplePrefix}
@@ -479,7 +600,7 @@ export function ClaimForm({ translations }: ClaimFormProps) {
<div className="card p-8 text-center"> <div className="card p-8 text-center">
<CheckCircle className="w-12 h-12 mx-auto mb-4 text-success" /> <CheckCircle className="w-12 h-12 mx-auto mb-4 text-success" />
<h2 className="text-xl font-semibold text-text-primary mb-2"> <h2 className="text-xl font-semibold text-text-primary mb-2">
{translations.addSuccess.title} {successTitle}
</h2> </h2>
<p className="text-text-secondary mb-4"> <p className="text-text-secondary mb-4">
{successDescription} {successDescription}
@@ -500,7 +621,7 @@ export function ClaimForm({ translations }: ClaimFormProps) {
</div> </div>
)} )}
<button <button
onClick={() => setAddSuccess(false)} onClick={() => { setAddSuccess(false); setAddReEnabled(false); }}
className="btn-secondary" className="btn-secondary"
> >
{translations.addSuccess.viewRequests} {translations.addSuccess.viewRequests}
@@ -535,6 +656,17 @@ export function ClaimForm({ translations }: ClaimFormProps) {
<Plus className="w-4 h-4" /> <Plus className="w-4 h-4" />
{translations.tabs.add} {translations.tabs.add}
</button> </button>
<button
onClick={() => handleTabChange('remove-repo')}
className={`flex items-center gap-2 px-4 py-3 text-sm font-medium border-b-2 transition-colors ${
activeTab === 'remove-repo'
? 'border-error text-error'
: 'border-transparent text-text-muted hover:text-text-primary'
}`}
>
<Trash2 className="w-4 h-4" />
{translations.tabs.removeRepo}
</button>
</div> </div>
{/* Remove Form */} {/* Remove Form */}
@@ -652,6 +784,79 @@ export function ClaimForm({ translations }: ClaimFormProps) {
</> </>
)} )}
{/* Remove Repository Form */}
{activeTab === 'remove-repo' && (
<div className="card p-6">
<form onSubmit={handleRepoRemovalSubmit} className="space-y-4">
<div>
<label
htmlFor="repoUrl"
className="block text-sm font-medium text-text-primary mb-2"
>
{translations.removeRepoForm.repoUrl}
</label>
<input
type="text"
id="repoUrl"
value={repoUrl}
onChange={(e) => setRepoUrl(e.target.value)}
placeholder={translations.removeRepoForm.repoUrlPlaceholder}
className="w-full px-4 py-2 border border-border rounded-lg bg-surface text-text-primary placeholder-text-muted focus:outline-none focus:ring-2 focus:ring-error"
dir="ltr"
required
/>
<p className="text-xs text-text-muted mt-1">
{translations.removeRepoForm.repoUrlHelp}
</p>
</div>
<div>
<label
htmlFor="repoReason"
className="block text-sm font-medium text-text-primary mb-2"
>
{translations.removeRepoForm.reason} <span className="text-text-muted">{translations.optional}</span>
</label>
<textarea
id="repoReason"
value={repoReason}
onChange={(e) => setRepoReason(e.target.value)}
placeholder={translations.removeRepoForm.reasonPlaceholder}
rows={3}
className="w-full px-4 py-2 border border-border rounded-lg bg-surface text-text-primary placeholder-text-muted focus:outline-none focus:ring-2 focus:ring-primary-500 resize-none"
/>
</div>
{repoError && (
<div className="p-3 bg-error-bg border border-error/30 rounded-lg">
<div className="flex items-center gap-2 text-error">
<AlertCircle className="w-4 h-4" />
<p className="text-sm">{repoError}</p>
</div>
</div>
)}
<button
type="submit"
disabled={isSubmittingRepo || !repoUrl.trim()}
className="w-full justify-center inline-flex items-center gap-2 px-4 py-2 bg-error text-white rounded-lg font-medium hover:bg-error/90 transition-colors disabled:opacity-50"
>
{isSubmittingRepo ? (
<>
<Loader2 className="w-4 h-4 animate-spin" />
{translations.removeRepoForm.submitting}
</>
) : (
<>
<Trash2 className="w-4 h-4" />
{translations.removeRepoForm.submit}
</>
)}
</button>
</form>
</div>
)}
{/* Add Form */} {/* Add Form */}
{activeTab === 'add' && ( {activeTab === 'add' && (
<> <>

View File

@@ -545,6 +545,8 @@
"browseAll": "Browse all skills", "browseAll": "Browse all skills",
"claimCta": "Is this your repository?", "claimCta": "Is this your repository?",
"claimButton": "Claim ownership", "claimButton": "Claim ownership",
"ownerManageCta": "You own these skills.",
"ownerManageButton": "Manage / Remove Skills",
"installCta": "Install the most popular skill from this owner:", "installCta": "Install the most popular skill from this owner:",
"stats": { "stats": {
"skills": "{count, plural, one {# skill} other {# skills}}", "skills": "{count, plural, one {# skill} other {# skills}}",
@@ -696,7 +698,8 @@
}, },
"tabs": { "tabs": {
"remove": "Remove Skill", "remove": "Remove Skill",
"add": "Add Skill" "add": "Add Skill",
"removeRepo": "Remove Repository"
}, },
"form": { "form": {
"skillId": "Skill ID", "skillId": "Skill ID",
@@ -732,7 +735,9 @@
"viewRequests": "View My Requests", "viewRequests": "View My Requests",
"foundSkillsIn": "Found skills in:", "foundSkillsIn": "Found skills in:",
"root": "(root)", "root": "(root)",
"andMore": "... and {count} more" "andMore": "... and {count} more",
"reEnabledTitle": "Repository Re-enabled",
"reEnabledDescription": "Your repository has been unblocked and will appear again after the next scan."
}, },
"error": { "error": {
"notOwner": "You are not the owner or admin of this repository. Only repository owners can request removal.", "notOwner": "You are not the owner or admin of this repository. Only repository owners can request removal.",
@@ -744,7 +749,8 @@
"invalidRepo": "The repository was not found or is not accessible. Please check the URL and ensure the repository is public.", "invalidRepo": "The repository was not found or is not accessible. Please check the URL and ensure the repository is public.",
"rateLimitExceeded": "GitHub API rate limit exceeded. Please try again in a few minutes.", "rateLimitExceeded": "GitHub API rate limit exceeded. Please try again in a few minutes.",
"networkTimeout": "Request timed out while checking the repository. Please try again.", "networkTimeout": "Request timed out while checking the repository. Please try again.",
"generic": "An error occurred. Please try again." "generic": "An error occurred. Please try again.",
"repoBlockedByOwner": "This repository was removed by its owner and cannot be re-added."
}, },
"myRequests": { "myRequests": {
"title": "My Requests", "title": "My Requests",
@@ -760,6 +766,25 @@
"showLess": "Show less", "showLess": "Show less",
"showAllPrefix": "Show all ", "showAllPrefix": "Show all ",
"showAllSuffix": " paths" "showAllSuffix": " paths"
},
"removeRepoForm": {
"repoUrl": "GitHub Repository URL or owner/repo",
"repoUrlPlaceholder": "https://github.com/owner/repo",
"repoUrlHelp": "All skills from this repository will be removed from SkillHub and it will not be re-indexed.",
"reason": "Reason for removal",
"reasonPlaceholder": "Why are you removing this repository?",
"submit": "Remove All Skills",
"submitting": "Removing..."
},
"removeRepoSuccess": {
"title": "Repository Removed",
"description": "All {count} skills from {repo} have been removed from SkillHub.",
"descriptionZero": "No active skills were found for this repository. It has been blocked from re-indexing."
},
"removeRepoError": {
"notOwner": "You are not the owner of this repository.",
"invalidRepo": "Repository not found on GitHub.",
"parseError": "Invalid format. Use https://github.com/owner/repo or owner/repo."
} }
}, },
"attribution": { "attribution": {

View File

@@ -545,6 +545,8 @@
"browseAll": "مرور همه مهارت‌ها", "browseAll": "مرور همه مهارت‌ها",
"claimCta": "آیا این مخزن شماست؟", "claimCta": "آیا این مخزن شماست؟",
"claimButton": "ثبت مالکیت", "claimButton": "ثبت مالکیت",
"ownerManageCta": "این مهارت‌ها متعلق به شما هستند.",
"ownerManageButton": "مدیریت / حذف مهارت‌ها",
"installCta": "نصب محبوب‌ترین مهارت از این مالک:", "installCta": "نصب محبوب‌ترین مهارت از این مالک:",
"stats": { "stats": {
"skills": "{count, plural, one {# مهارت} other {# مهارت}}", "skills": "{count, plural, one {# مهارت} other {# مهارت}}",
@@ -696,7 +698,8 @@
}, },
"tabs": { "tabs": {
"remove": "حذف مهارت", "remove": "حذف مهارت",
"add": "افزودن مهارت" "add": "افزودن مهارت",
"removeRepo": "حذف مخزن"
}, },
"form": { "form": {
"skillId": "شناسه مهارت", "skillId": "شناسه مهارت",
@@ -732,7 +735,9 @@
"viewRequests": "مشاهده درخواست‌ها", "viewRequests": "مشاهده درخواست‌ها",
"foundSkillsIn": "مهارت‌ها در این مسیرها یافت شدند:", "foundSkillsIn": "مهارت‌ها در این مسیرها یافت شدند:",
"root": "(root)", "root": "(root)",
"andMore": "... و {count} مورد دیگر" "andMore": "... و {count} مورد دیگر",
"reEnabledTitle": "مخزن فعال‌سازی مجدد شد",
"reEnabledDescription": "مخزن شما رفع انسداد شد و پس از اسکن بعدی دوباره نمایش داده خواهد شد."
}, },
"error": { "error": {
"notOwner": "شما مالک یا ادمین این مخزن نیستید. فقط مالکان مخزن می‌توانند درخواست حذف دهند.", "notOwner": "شما مالک یا ادمین این مخزن نیستید. فقط مالکان مخزن می‌توانند درخواست حذف دهند.",
@@ -744,7 +749,8 @@
"invalidRepo": "مخزن یافت نشد یا قابل دسترسی نیست. لطفاً URL را بررسی کنید و اطمینان حاصل کنید که مخزن عمومی است.", "invalidRepo": "مخزن یافت نشد یا قابل دسترسی نیست. لطفاً URL را بررسی کنید و اطمینان حاصل کنید که مخزن عمومی است.",
"rateLimitExceeded": "محدودیت API GitHub تمام شده است. لطفاً چند دقیقه بعد دوباره تلاش کنید.", "rateLimitExceeded": "محدودیت API GitHub تمام شده است. لطفاً چند دقیقه بعد دوباره تلاش کنید.",
"networkTimeout": "هنگام بررسی مخزن زمان اتصال تمام شد. لطفاً دوباره تلاش کنید.", "networkTimeout": "هنگام بررسی مخزن زمان اتصال تمام شد. لطفاً دوباره تلاش کنید.",
"generic": "خطایی رخ داد. لطفاً دوباره تلاش کنید." "generic": "خطایی رخ داد. لطفاً دوباره تلاش کنید.",
"repoBlockedByOwner": "این مخزن توسط مالکش حذف شده و قابل افزودن مجدد نیست."
}, },
"myRequests": { "myRequests": {
"title": "درخواست‌های من", "title": "درخواست‌های من",
@@ -760,6 +766,25 @@
"showLess": "نمایش کمتر", "showLess": "نمایش کمتر",
"showAllPrefix": "نمایش همه ", "showAllPrefix": "نمایش همه ",
"showAllSuffix": " مسیر" "showAllSuffix": " مسیر"
},
"removeRepoForm": {
"repoUrl": "آدرس مخزن گیت‌هاب یا owner/repo",
"repoUrlPlaceholder": "https://github.com/owner/repo",
"repoUrlHelp": "تمام مهارت‌های این مخزن از SkillHub حذف شده و دیگر ایندکس نخواهد شد.",
"reason": "دلیل حذف",
"reasonPlaceholder": "چرا این مخزن را حذف می‌کنید؟",
"submit": "حذف تمام مهارت‌ها",
"submitting": "در حال حذف..."
},
"removeRepoSuccess": {
"title": "مخزن حذف شد",
"description": "تمام {count} مهارت از {repo} از SkillHub حذف شدند.",
"descriptionZero": "هیچ مهارت فعالی برای این مخزن یافت نشد. مخزن از ایندکس‌سازی مجدد مسدود شده است."
},
"removeRepoError": {
"notOwner": "شما مالک این مخزن نیستید.",
"invalidRepo": "مخزن در گیت‌هاب یافت نشد.",
"parseError": "فرمت نادرست. از https://github.com/owner/repo یا owner/repo استفاده کنید."
} }
}, },
"attribution": { "attribution": {

View File

@@ -83,6 +83,9 @@ export const config = {
'/(en|fa)/:path*', '/(en|fa)/:path*',
// Match API routes (for CSRF protection) // Match API routes (for CSRF protection)
'/api/:path*', '/api/:path*',
// Match skill paths explicitly (repo names like "next.js" contain dots
// which would be excluded by the catch-all pattern below)
'/skill/:path*',
// Match other paths (exclude static files and Next.js internals) // Match other paths (exclude static files and Next.js internals)
'/((?!_next|_vercel|.*\\..*).*)', '/((?!_next|_vercel|.*\\..*).*)',
], ],

View File

@@ -43,6 +43,20 @@ Sentry.init({
/Object Not Found Matching Id/, /Object Not Found Matching Id/,
// ResizeObserver loop limit — browser quirk, not actionable // ResizeObserver loop limit — browser quirk, not actionable
"ResizeObserver", "ResizeObserver",
// DOM manipulation by browser extensions (translate, font, etc.) conflicts with React
"NotFoundError: Failed to execute 'removeChild' on 'Node'",
"NotFoundError: Failed to execute 'insertBefore' on 'Node'",
// Browser extension internal errors (crypto wallets, etc.)
/func .* not found/,
],
// Ignore errors originating from browser extensions or injected scripts
denyUrls: [
/extensions\//i,
/^chrome:\/\//i,
/^chrome-extension:\/\//i,
/^moz-extension:\/\//i,
/inpage\.js/,
], ],
beforeSend(event, hint) { beforeSend(event, hint) {

View File

@@ -121,8 +121,14 @@ services:
volumes: volumes:
postgres_data: postgres_data:
name: skillhub-public_postgres_data
external: true
redis_data: redis_data:
name: skillhub-public_redis_data
external: true
meilisearch_data: meilisearch_data:
name: skillhub-public_meilisearch_data
external: true
networks: networks:
default: default:

View File

@@ -6,6 +6,7 @@ import {
ratingQueries, ratingQueries,
installationQueries, installationQueries,
favoriteQueries, favoriteQueries,
discoveredRepoQueries,
} from './queries.js'; } from './queries.js';
import { import {
createTestSkill, createTestSkill,
@@ -22,13 +23,14 @@ import {
// Mock the schema imports // Mock the schema imports
vi.mock('./schema.js', () => ({ vi.mock('./schema.js', () => ({
skills: { id: 'id', name: 'name', description: 'description', githubStars: 'github_stars', downloadCount: 'download_count', rating: 'rating', updatedAt: 'updated_at', isFeatured: 'is_featured', isVerified: 'is_verified', securityScore: 'security_score', viewCount: 'view_count', ratingCount: 'rating_count', ratingSum: 'rating_sum' }, skills: { id: 'id', name: 'name', description: 'description', githubStars: 'github_stars', downloadCount: 'download_count', rating: 'rating', updatedAt: 'updated_at', isFeatured: 'is_featured', isVerified: 'is_verified', securityScore: 'security_score', viewCount: 'view_count', ratingCount: 'rating_count', ratingSum: 'rating_sum', githubOwner: 'github_owner', githubRepo: 'github_repo', isBlocked: 'is_blocked', isOwnerClaimed: 'is_owner_claimed', isDuplicate: 'is_duplicate' },
categories: { id: 'id', name: 'name', slug: 'slug', sortOrder: 'sort_order', skillCount: 'skill_count' }, categories: { id: 'id', name: 'name', slug: 'slug', sortOrder: 'sort_order', skillCount: 'skill_count' },
skillCategories: { skillId: 'skill_id', categoryId: 'category_id' }, skillCategories: { skillId: 'skill_id', categoryId: 'category_id' },
users: { id: 'id', githubId: 'github_id', username: 'username', avatarUrl: 'avatar_url' }, users: { id: 'id', githubId: 'github_id', username: 'username', avatarUrl: 'avatar_url' },
ratings: { id: 'id', skillId: 'skill_id', userId: 'user_id', rating: 'rating', createdAt: 'created_at' }, ratings: { id: 'id', skillId: 'skill_id', userId: 'user_id', rating: 'rating', createdAt: 'created_at' },
installations: { id: 'id', skillId: 'skill_id', platform: 'platform', method: 'method' }, installations: { id: 'id', skillId: 'skill_id', platform: 'platform', method: 'method' },
favorites: { userId: 'user_id', skillId: 'skill_id', createdAt: 'created_at' }, favorites: { userId: 'user_id', skillId: 'skill_id', createdAt: 'created_at' },
discoveredRepos: { id: 'id', isBlocked: 'is_blocked', isArchived: 'is_archived', lastScanned: 'last_scanned', githubStars: 'github_stars', discoveredVia: 'discovered_via', owner: 'owner', repo: 'repo', githubForks: 'github_forks', defaultBranch: 'default_branch', hasSkillMd: 'has_skill_md', skillCount: 'skill_count', scanError: 'scan_error', updatedAt: 'updated_at' },
})); }));
// Helper to create a chainable mock // Helper to create a chainable mock
@@ -659,3 +661,127 @@ describe('favoriteQueries', () => {
}); });
}); });
}); });
describe('skillQueries (repo block/unblock)', () => {
describe('blockByRepo', () => {
it('should call update with isBlocked: true for matching owner/repo', async () => {
const mockDb = createMockDb();
await skillQueries.blockByRepo(mockDb as any, 'rawveg', 'skillsforge-marketplace');
expect(mockDb.update).toHaveBeenCalled();
});
});
describe('unblockByRepo', () => {
it('should call update with isBlocked: false for matching owner/repo', async () => {
const mockDb = createMockDb();
await skillQueries.unblockByRepo(mockDb as any, 'rawveg', 'skillsforge-marketplace');
expect(mockDb.update).toHaveBeenCalled();
});
});
describe('setOwnerClaimed', () => {
it('should call update with isOwnerClaimed: true for matching owner/repo', async () => {
const mockDb = createMockDb();
await skillQueries.setOwnerClaimed(mockDb as any, 'rawveg', 'skillsforge-marketplace');
expect(mockDb.update).toHaveBeenCalled();
});
});
describe('countByRepo', () => {
it('should return count of non-blocked skills for owner/repo', async () => {
const mockDb = createMockDb([{ count: 16 }]);
const result = await skillQueries.countByRepo(mockDb as any, 'rawveg', 'skillsforge-marketplace');
expect(result).toBe(16);
expect(mockDb.select).toHaveBeenCalled();
});
it('should return 0 when no matching skills', async () => {
const mockDb = createMockDb([{ count: 0 }]);
const result = await skillQueries.countByRepo(mockDb as any, 'unknown', 'unknown-repo');
expect(result).toBe(0);
});
it('should return 0 when result is empty', async () => {
const mockDb = createMockDb([]);
const result = await skillQueries.countByRepo(mockDb as any, 'owner', 'repo');
expect(result).toBe(0);
});
});
});
describe('discoveredRepoQueries', () => {
describe('blockRepo', () => {
it('should call update with isBlocked: true for the repo id', async () => {
const mockDb = createMockDb();
await discoveredRepoQueries.blockRepo(mockDb as any, 'rawveg/skillsforge-marketplace');
expect(mockDb.update).toHaveBeenCalled();
});
});
describe('unblockRepo', () => {
it('should call update with isBlocked: false for the repo id', async () => {
const mockDb = createMockDb();
await discoveredRepoQueries.unblockRepo(mockDb as any, 'rawveg/skillsforge-marketplace');
expect(mockDb.update).toHaveBeenCalled();
});
});
describe('getNeedingScanning', () => {
it('should return repos that need scanning', async () => {
const repos = [
{ id: 'owner/repo1', isBlocked: false },
{ id: 'owner/repo2', isBlocked: false },
];
const mockDb = createMockDb(repos);
const result = await discoveredRepoQueries.getNeedingScanning(mockDb as any);
expect(result).toEqual(repos);
expect(mockDb.select).toHaveBeenCalled();
});
it('should return empty array when no repos need scanning', async () => {
const mockDb = createMockDb([]);
const result = await discoveredRepoQueries.getNeedingScanning(mockDb as any);
expect(result).toEqual([]);
});
});
describe('getById', () => {
it('should return repo when found', async () => {
const repo = { id: 'rawveg/skillsforge-marketplace', isBlocked: true };
const mockDb = createMockDb([repo]);
const result = await discoveredRepoQueries.getById(mockDb as any, 'rawveg/skillsforge-marketplace');
expect(result).toEqual(repo);
expect(mockDb.select).toHaveBeenCalled();
});
it('should return null when repo not found', async () => {
const mockDb = createMockDb([]);
const result = await discoveredRepoQueries.getById(mockDb as any, 'nonexistent/repo');
expect(result).toBeNull();
});
});
});

View File

@@ -36,7 +36,7 @@ type DB = PostgresJsDatabase<typeof schema>;
* Skills with skill_type=NULL (newly crawled, not yet curated) are included * Skills with skill_type=NULL (newly crawled, not yet curated) are included
* so they don't disappear between curate.mjs runs. * so they don't disappear between curate.mjs runs.
*/ */
const browseReadyFilter = sql`(${skills.isDuplicate} = false)`; const browseReadyFilter = sql`(${skills.isDuplicate} = false OR ${skills.isOwnerClaimed} = true) AND ${skills.isStale} = false`;
/** Minimum repo age gap (in days) to trust repo_created_at over stars for duplicate detection */ /** Minimum repo age gap (in days) to trust repo_created_at over stars for duplicate detection */
const REPO_AGE_THRESHOLD_DAYS = 75; const REPO_AGE_THRESHOLD_DAYS = 75;
@@ -688,6 +688,10 @@ export const skillQueries = {
: (skill.securityStatus && skill.qualityScore != null) : (skill.securityStatus && skill.qualityScore != null)
? { reviewStatus: sql`CASE WHEN ${skills.reviewStatus} IS NULL OR ${skills.reviewStatus} = 'unreviewed' THEN 'auto-scored' ELSE ${skills.reviewStatus} END` } ? { reviewStatus: sql`CASE WHEN ${skills.reviewStatus} IS NULL OR ${skills.reviewStatus} = 'unreviewed' THEN 'auto-scored' ELSE ${skills.reviewStatus} END` }
: {}), : {}),
// Clear stale state on successful re-index (skill is accessible again)
isStale: false,
staleSince: null,
staleCheckCount: 0,
}, },
}) })
.returning(); .returning();
@@ -759,6 +763,44 @@ export const skillQueries = {
await db.update(skills).set({ isBlocked: false }).where(eq(skills.id, id)); await db.update(skills).set({ isBlocked: false }).where(eq(skills.id, id));
}, },
/**
* Mark all skills from a repo as owner-claimed (exempt from duplicate hiding)
*/
setOwnerClaimed: async (db: DB, owner: string, repo: string) => {
await db.update(skills)
.set({ isOwnerClaimed: true })
.where(and(eq(skills.githubOwner, owner), eq(skills.githubRepo, repo)));
},
/**
* Block all skills from a repo (owner requested repo-level removal)
*/
blockByRepo: async (db: DB, owner: string, repo: string) => {
await db.update(skills)
.set({ isBlocked: true })
.where(and(eq(skills.githubOwner, owner), eq(skills.githubRepo, repo)));
},
/**
* Unblock all skills from a repo (owner re-enabled their repository)
*/
unblockByRepo: async (db: DB, owner: string, repo: string) => {
await db.update(skills)
.set({ isBlocked: false })
.where(and(eq(skills.githubOwner, owner), eq(skills.githubRepo, repo)));
},
/**
* Count non-blocked skills in a repo
*/
countByRepo: async (db: DB, owner: string, repo: string): Promise<number> => {
const result = await db
.select({ count: sql<number>`count(*)::int` })
.from(skills)
.where(and(eq(skills.githubOwner, owner), eq(skills.githubRepo, repo), eq(skills.isBlocked, false)));
return result[0]?.count ?? 0;
},
/** /**
* Check if a skill is blocked * Check if a skill is blocked
*/ */
@@ -771,6 +813,92 @@ export const skillQueries = {
return result[0]?.isBlocked ?? false; return result[0]?.isBlocked ?? false;
}, },
/**
* Increment stale check count. After 3 consecutive 404s, mark skill as stale.
* Sets stale_since on first detection. Idempotent if skill doesn't exist.
*/
incrementStaleCheck: async (db: DB, id: string) => {
const STALE_THRESHOLD = 3;
const existing = await db
.select({
staleCheckCount: skills.staleCheckCount,
staleSince: skills.staleSince,
})
.from(skills)
.where(eq(skills.id, id))
.limit(1);
if (!existing[0]) return;
const newCount = (existing[0].staleCheckCount ?? 0) + 1;
const isNowStale = newCount >= STALE_THRESHOLD;
await db.update(skills).set({
staleCheckCount: newCount,
staleSince: existing[0].staleSince ?? new Date(),
isStale: isNowStale,
}).where(eq(skills.id, id));
},
/**
* Clear stale state when a skill is successfully re-fetched from GitHub
*/
clearStaleState: async (db: DB, id: string) => {
await db.update(skills).set({
isStale: false,
staleSince: null,
staleCheckCount: 0,
}).where(eq(skills.id, id));
},
/**
* Get stale skills (for admin/reporting)
*/
getStaleSkills: async (db: DB, limit = 100, offset = 0) => {
return db
.select()
.from(skills)
.where(eq(skills.isStale, true))
.orderBy(desc(skills.staleSince))
.limit(limit)
.offset(offset);
},
/**
* Count stale skills
*/
countStale: async (db: DB): Promise<number> => {
const result = await db
.select({ count: sql<number>`count(*)::int` })
.from(skills)
.where(eq(skills.isStale, true));
return result[0]?.count ?? 0;
},
/**
* Get skills to check for staleness (ordered by oldest scanned first)
*/
getSkillsToStaleCheck: async (db: DB, limit = 500) => {
return db
.select({
id: skills.id,
githubOwner: skills.githubOwner,
githubRepo: skills.githubRepo,
skillPath: skills.skillPath,
branch: skills.branch,
sourceFormat: skills.sourceFormat,
staleCheckCount: skills.staleCheckCount,
})
.from(skills)
.where(and(
eq(skills.isBlocked, false),
eq(skills.isStale, false),
))
.orderBy(asc(skills.lastScanned))
.limit(limit);
},
/** /**
* Get cached files for a skill * Get cached files for a skill
* Returns null if cache doesn't exist or is stale (commitSha mismatch) * Returns null if cache doesn't exist or is stale (commitSha mismatch)
@@ -1266,6 +1394,7 @@ export async function runPostCrawlCuration(db: DB): Promise<{ classified: number
UPDATE skills s SET is_duplicate = true UPDATE skills s SET is_duplicate = true
WHERE s.is_blocked = false WHERE s.is_blocked = false
AND s.is_duplicate = false AND s.is_duplicate = false
AND s.is_owner_claimed = false
AND s.content_hash IS NOT NULL AND s.content_hash IS NOT NULL
AND EXISTS ( AND EXISTS (
SELECT 1 FROM skills s2 SELECT 1 FROM skills s2
@@ -1623,6 +1752,7 @@ export const discoveredRepoQueries = {
conditions.push(sql`${discoveredRepos.lastScanned} IS NULL`); conditions.push(sql`${discoveredRepos.lastScanned} IS NULL`);
} }
conditions.push(eq(discoveredRepos.isArchived, false)); conditions.push(eq(discoveredRepos.isArchived, false));
conditions.push(eq(discoveredRepos.isBlocked, false));
return db return db
.select() .select()
@@ -1632,6 +1762,20 @@ export const discoveredRepoQueries = {
.limit(limit); .limit(limit);
}, },
/**
* Block a repo from re-indexing (owner requested removal of all skills)
*/
blockRepo: async (db: DB, id: string) => {
await db.update(discoveredRepos).set({ isBlocked: true }).where(eq(discoveredRepos.id, id));
},
/**
* Unblock a repo (owner re-enabled it after previously removing it)
*/
unblockRepo: async (db: DB, id: string) => {
await db.update(discoveredRepos).set({ isBlocked: false }).where(eq(discoveredRepos.id, id));
},
/** /**
* Mark a repo as scanned * Mark a repo as scanned
*/ */
@@ -2511,6 +2655,7 @@ export const skillReviewQueries = {
priorityReReview?: boolean; priorityReReview?: boolean;
reReviewAll?: boolean; reReviewAll?: boolean;
ownerLimit?: number; ownerLimit?: number;
currentReviewVersion?: number;
} = {} } = {}
) => { ) => {
const { const {
@@ -2521,6 +2666,7 @@ export const skillReviewQueries = {
priorityReReview = false, priorityReReview = false,
reReviewAll = false, reReviewAll = false,
ownerLimit = 0, ownerLimit = 0,
currentReviewVersion = 0,
} = options; } = options;
const conditions = [ const conditions = [
@@ -2539,6 +2685,16 @@ export const skillReviewQueries = {
conditions.push( conditions.push(
sql`${skills.reviewStatus} IN ('ai-reviewed', 'needs-re-review', 'auto-scored')` sql`${skills.reviewStatus} IN ('ai-reviewed', 'needs-re-review', 'auto-scored')`
); );
// Skip skills already reviewed at the current version
if (currentReviewVersion > 0) {
conditions.push(
sql`NOT EXISTS (
SELECT 1 FROM skill_reviews sr
WHERE sr.skill_id = ${skills.id}
AND sr.review_version >= ${currentReviewVersion}
)`
);
}
} else if (priorityReReview) { } else if (priorityReReview) {
conditions.push(eq(skills.reviewStatus, 'needs-re-review')); conditions.push(eq(skills.reviewStatus, 'needs-re-review'));
} else { } else {
@@ -2570,6 +2726,12 @@ export const skillReviewQueries = {
branch: skills.branch, branch: skills.branch,
}; };
// Re-review-all: prioritize already-reviewed skills first, then auto-scored
// Use sql.raw() for the ORDER BY to avoid parameterization issues in db.execute()
const orderBySql = reReviewAll
? sql`CASE WHEN review_status IN ('ai-reviewed', 'needs-re-review') THEN 0 ELSE 1 END, quality_score DESC NULLS LAST, github_stars DESC NULLS LAST`
: sql`quality_score DESC NULLS LAST, github_stars DESC NULLS LAST`;
// Owner-capped batch: limit skills per github_owner for diversity // Owner-capped batch: limit skills per github_owner for diversity
if (ownerLimit > 0) { if (ownerLimit > 0) {
const whereClause = and(...conditions); const whereClause = and(...conditions);
@@ -2580,7 +2742,7 @@ export const skillReviewQueries = {
github_owner, github_repo, skill_path, branch, github_owner, github_repo, skill_path, branch,
ROW_NUMBER() OVER ( ROW_NUMBER() OVER (
PARTITION BY github_owner PARTITION BY github_owner
ORDER BY quality_score DESC NULLS LAST, github_stars DESC NULLS LAST ORDER BY ${orderBySql}
) AS owner_rank ) AS owner_rank
FROM skills FROM skills
WHERE ${whereClause} WHERE ${whereClause}
@@ -2593,18 +2755,23 @@ export const skillReviewQueries = {
skill_path AS "skillPath", branch skill_path AS "skillPath", branch
FROM ranked FROM ranked
WHERE owner_rank <= ${ownerLimit} WHERE owner_rank <= ${ownerLimit}
ORDER BY quality_score DESC NULLS LAST, github_stars DESC NULLS LAST ORDER BY ${orderBySql}
LIMIT ${batchSize} LIMIT ${batchSize}
OFFSET ${offset} OFFSET ${offset}
`); `);
return [...results]; return [...results];
} }
// Build order: re-review priority first (if applicable), then quality
const orderClauses = reReviewAll
? [sql`CASE WHEN ${skills.reviewStatus} IN ('ai-reviewed', 'needs-re-review') THEN 0 ELSE 1 END`, desc(skills.qualityScore), desc(skills.githubStars)]
: [desc(skills.qualityScore), desc(skills.githubStars)];
return db return db
.select(selectFields) .select(selectFields)
.from(skills) .from(skills)
.where(and(...conditions)) .where(and(...conditions))
.orderBy(desc(skills.qualityScore), desc(skills.githubStars)) .orderBy(...orderClauses)
.limit(batchSize) .limit(batchSize)
.offset(offset); .offset(offset);
}, },

View File

@@ -67,6 +67,9 @@ export const skills = pgTable(
isFeatured: boolean('is_featured').default(false), isFeatured: boolean('is_featured').default(false),
isBlocked: boolean('is_blocked').default(false), // Blocked from re-indexing (owner requested removal) isBlocked: boolean('is_blocked').default(false), // Blocked from re-indexing (owner requested removal)
isDeprecated: boolean('is_deprecated').default(false), // Auto-detected from raw_content (DEPRECATED/ARCHIVED markers) isDeprecated: boolean('is_deprecated').default(false), // Auto-detected from raw_content (DEPRECATED/ARCHIVED markers)
isStale: boolean('is_stale').default(false), // Skill files no longer accessible on GitHub (confirmed after 3 consecutive 404s)
staleSince: timestamp('stale_since'), // When staleness was first detected
staleCheckCount: integer('stale_check_count').default(0), // Consecutive 404 count (threshold: 3)
lastScanned: timestamp('last_scanned'), lastScanned: timestamp('last_scanned'),
// Curation (populated by batch scripts, not crawler) // Curation (populated by batch scripts, not crawler)
@@ -79,6 +82,7 @@ export const skills = pgTable(
}>(), }>(),
skillType: text('skill_type').$type<'standalone' | 'project-bound' | 'collection' | 'aggregator'>(), skillType: text('skill_type').$type<'standalone' | 'project-bound' | 'collection' | 'aggregator'>(),
isDuplicate: boolean('is_duplicate').default(false), isDuplicate: boolean('is_duplicate').default(false),
isOwnerClaimed: boolean('is_owner_claimed').default(false), // Owner explicitly submitted via Add Skill — exempt from duplicate hiding
canonicalSkillId: text('canonical_skill_id'), // points to the "original" if this is a duplicate canonicalSkillId: text('canonical_skill_id'), // points to the "original" if this is a duplicate
repoSkillCount: integer('repo_skill_count'), // cached count of skills in same repo repoSkillCount: integer('repo_skill_count'), // cached count of skills in same repo
reviewStatus: text('review_status').default('unreviewed'), // unreviewed → auto-scored → ai-reviewed → verified (or needs-re-review) reviewStatus: text('review_status').default('unreviewed'), // unreviewed → auto-scored → ai-reviewed → verified (or needs-re-review)
@@ -124,6 +128,7 @@ export const skills = pgTable(
skillTypeIdx: index('idx_skills_type').on(table.skillType), skillTypeIdx: index('idx_skills_type').on(table.skillType),
duplicateIdx: index('idx_skills_duplicate').on(table.isDuplicate), duplicateIdx: index('idx_skills_duplicate').on(table.isDuplicate),
contentHashIdx: index('idx_skills_content_hash').on(table.contentHash), contentHashIdx: index('idx_skills_content_hash').on(table.contentHash),
staleIdx: index('idx_skills_stale').on(table.isStale),
}) })
); );
@@ -303,6 +308,7 @@ export const discoveredRepos = pgTable(
githubForks: integer('github_forks').default(0), githubForks: integer('github_forks').default(0),
defaultBranch: text('default_branch').default('main'), defaultBranch: text('default_branch').default('main'),
isArchived: boolean('is_archived').default(false), isArchived: boolean('is_archived').default(false),
isBlocked: boolean('is_blocked').default(false), // Owner requested removal — skip re-indexing
scanError: text('scan_error'), // Last scan error if any scanError: text('scan_error'), // Last scan error if any
createdAt: timestamp('created_at').defaultNow().notNull(), createdAt: timestamp('created_at').defaultNow().notNull(),
updatedAt: timestamp('updated_at').defaultNow().notNull(), updatedAt: timestamp('updated_at').defaultNow().notNull(),

View File

@@ -164,6 +164,7 @@ CREATE TABLE IF NOT EXISTS discovered_repos (
github_forks INTEGER DEFAULT 0, github_forks INTEGER DEFAULT 0,
default_branch TEXT DEFAULT 'main', default_branch TEXT DEFAULT 'main',
is_archived BOOLEAN DEFAULT FALSE, is_archived BOOLEAN DEFAULT FALSE,
is_blocked BOOLEAN NOT NULL DEFAULT FALSE,
scan_error TEXT, scan_error TEXT,
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL, created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL
@@ -471,13 +472,17 @@ ALTER TABLE skills ADD COLUMN IF NOT EXISTS quality_score INTEGER;
ALTER TABLE skills ADD COLUMN IF NOT EXISTS quality_details JSONB; ALTER TABLE skills ADD COLUMN IF NOT EXISTS quality_details JSONB;
ALTER TABLE skills ADD COLUMN IF NOT EXISTS skill_type TEXT; -- 'standalone', 'project-bound', 'collection', 'aggregator' ALTER TABLE skills ADD COLUMN IF NOT EXISTS skill_type TEXT; -- 'standalone', 'project-bound', 'collection', 'aggregator'
ALTER TABLE skills ADD COLUMN IF NOT EXISTS is_duplicate BOOLEAN DEFAULT FALSE; ALTER TABLE skills ADD COLUMN IF NOT EXISTS is_duplicate BOOLEAN DEFAULT FALSE;
ALTER TABLE skills ADD COLUMN IF NOT EXISTS is_owner_claimed BOOLEAN NOT NULL DEFAULT FALSE;
ALTER TABLE skills ADD COLUMN IF NOT EXISTS canonical_skill_id TEXT; -- points to original if duplicate ALTER TABLE skills ADD COLUMN IF NOT EXISTS canonical_skill_id TEXT; -- points to original if duplicate
ALTER TABLE skills ADD COLUMN IF NOT EXISTS repo_skill_count INTEGER; -- cached count of skills in repo ALTER TABLE skills ADD COLUMN IF NOT EXISTS repo_skill_count INTEGER; -- cached count of skills in repo
ALTER TABLE discovered_repos ADD COLUMN IF NOT EXISTS is_blocked BOOLEAN NOT NULL DEFAULT FALSE;
CREATE INDEX IF NOT EXISTS idx_skills_quality ON skills(quality_score DESC NULLS LAST); CREATE INDEX IF NOT EXISTS idx_skills_quality ON skills(quality_score DESC NULLS LAST);
CREATE INDEX IF NOT EXISTS idx_skills_type ON skills(skill_type); CREATE INDEX IF NOT EXISTS idx_skills_type ON skills(skill_type);
CREATE INDEX IF NOT EXISTS idx_skills_duplicate ON skills(is_duplicate); CREATE INDEX IF NOT EXISTS idx_skills_duplicate ON skills(is_duplicate);
CREATE INDEX IF NOT EXISTS idx_skills_content_hash ON skills(content_hash); CREATE INDEX IF NOT EXISTS idx_skills_content_hash ON skills(content_hash);
CREATE INDEX IF NOT EXISTS idx_skills_owner_claimed ON skills(is_owner_claimed) WHERE is_owner_claimed = TRUE;
CREATE INDEX IF NOT EXISTS idx_discovered_repos_blocked ON discovered_repos(is_blocked) WHERE is_blocked = TRUE;
-- Review pipeline columns (Phase 4+5 - February 2026) -- Review pipeline columns (Phase 4+5 - February 2026)
ALTER TABLE skills ADD COLUMN IF NOT EXISTS review_status TEXT DEFAULT 'unreviewed'; ALTER TABLE skills ADD COLUMN IF NOT EXISTS review_status TEXT DEFAULT 'unreviewed';
@@ -557,6 +562,12 @@ CREATE INDEX IF NOT EXISTS idx_reviews_skill ON skill_reviews(skill_id);
CREATE INDEX IF NOT EXISTS idx_reviews_score ON skill_reviews(ai_score); CREATE INDEX IF NOT EXISTS idx_reviews_score ON skill_reviews(ai_score);
CREATE INDEX IF NOT EXISTS idx_reviews_blog ON skill_reviews(blog_worthy) WHERE blog_worthy = true; CREATE INDEX IF NOT EXISTS idx_reviews_blog ON skill_reviews(blog_worthy) WHERE blog_worthy = true;
-- Stale skill detection (March 2026)
ALTER TABLE skills ADD COLUMN IF NOT EXISTS is_stale BOOLEAN DEFAULT FALSE;
ALTER TABLE skills ADD COLUMN IF NOT EXISTS stale_since TIMESTAMP WITH TIME ZONE;
ALTER TABLE skills ADD COLUMN IF NOT EXISTS stale_check_count INTEGER DEFAULT 0;
CREATE INDEX IF NOT EXISTS idx_skills_stale ON skills(is_stale) WHERE is_stale = TRUE;
-- Grant permissions -- Grant permissions
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO postgres; GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO postgres;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO postgres; GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO postgres;

View File

@@ -7,7 +7,7 @@
import { INSTRUCTION_FILE_PATTERNS, type SourceFormat } from 'skillhub-core'; import { INSTRUCTION_FILE_PATTERNS, type SourceFormat } from 'skillhub-core';
import { scheduleFullCrawl, scheduleIncrementalCrawl, getQueueStats, getQueue } from './queue.js'; import { scheduleFullCrawl, scheduleIncrementalCrawl, getQueueStats, getQueue } from './queue.js';
import { syncAllSkillsToMeilisearch, checkMeilisearchHealth } from './meilisearch-sync.js'; import { syncAllSkillsToMeilisearch, checkMeilisearchHealth } from './meilisearch-sync.js';
import { createDb, skillQueries, categoryQueries, discoveredRepoQueries, awesomeListQueries, addRequestQueries, userQueries, sql } from '@skillhub/db'; import { createDb, skillQueries, categoryQueries, discoveredRepoQueries, awesomeListQueries, addRequestQueries, userQueries, skills, sql } from '@skillhub/db';
import { createStrategyOrchestrator, createDeepScanCrawler, createAwesomeListCrawler, createPopularReposCrawler, createCommitsSearchCrawler } from './strategies/index.js'; import { createStrategyOrchestrator, createDeepScanCrawler, createAwesomeListCrawler, createPopularReposCrawler, createCommitsSearchCrawler } from './strategies/index.js';
import { createCrawler } from './crawler.js'; import { createCrawler } from './crawler.js';
import { indexSkill } from './skill-indexer.js'; import { indexSkill } from './skill-indexer.js';
@@ -870,6 +870,9 @@ async function main() {
status: 'indexed', status: 'indexed',
indexedSkillId: allSkillIds.join(','), indexedSkillId: allSkillIds.join(','),
}); });
// Mark all skills from this repo as owner-claimed so they're
// exempt from duplicate hiding on the owner page and browse pages
await skillQueries.setOwnerClaimed(addDb, owner, repo);
const newCount = indexedSkillIds.length; const newCount = indexedSkillIds.length;
const existingCount = existingSkillIds.length; const existingCount = existingSkillIds.length;
if (newCount > 0 && existingCount > 0) { if (newCount > 0 && existingCount > 0) {
@@ -926,6 +929,170 @@ async function main() {
break; break;
} }
case 'add-repo': {
const input = process.argv[3];
if (!input) {
console.error('Usage: node dist/crawl.js add-repo <owner/repo or https://github.com/owner/repo>');
process.exit(1);
}
let arOwner: string;
let arRepo: string;
try {
if (input.startsWith('https://') || input.startsWith('http://')) {
const arUrl = new URL(input);
const arParts = arUrl.pathname.split('/').filter(Boolean);
arOwner = arParts[0];
arRepo = arParts[1];
} else {
const arParts = input.split('/').filter(Boolean);
arOwner = arParts[0];
arRepo = arParts[1];
}
} catch {
console.error('Could not parse owner/repo from input:', input);
process.exit(1);
}
if (!arOwner! || !arRepo!) {
console.error('Could not parse owner/repo from input:', input);
process.exit(1);
}
console.log(`Validating ${arOwner}/${arRepo} on GitHub...`);
const arGhRes = await fetch(`https://api.github.com/repos/${arOwner}/${arRepo}`, {
headers: {
Authorization: `token ${process.env.GITHUB_TOKEN}`,
'User-Agent': 'SkillHub',
Accept: 'application/vnd.github.v3+json',
},
signal: AbortSignal.timeout(10000),
});
if (!arGhRes.ok) {
console.error(`Repo ${arOwner}/${arRepo} not found on GitHub (HTTP ${arGhRes.status})`);
process.exit(1);
}
const arGhData = await arGhRes.json() as {
stargazers_count: number;
forks_count: number;
default_branch: string;
archived: boolean;
};
const arDb = createDb(process.env.DATABASE_URL);
await discoveredRepoQueries.upsert(arDb, {
id: `${arOwner}/${arRepo}`,
owner: arOwner,
repo: arRepo,
discoveredVia: 'manual',
sourceUrl: `https://github.com/${arOwner}/${arRepo}`,
githubStars: arGhData.stargazers_count ?? 0,
githubForks: arGhData.forks_count ?? 0,
defaultBranch: arGhData.default_branch ?? 'main',
isArchived: arGhData.archived ?? false,
});
console.log(`✅ Added ${arOwner}/${arRepo} to discovered_repos.`);
console.log(` Run 'deep-scan' to index skills from this repository.`);
break;
}
case 'stale-check': {
console.log('Checking for stale skills (removed from GitHub)...\n');
const staleDb = createDb(process.env.DATABASE_URL);
const staleCrawler = createCrawler();
// Parse --limit=N option
const staleLimitArg = process.argv.find(a => a.startsWith('--limit='));
const staleLimit = staleLimitArg ? parseInt(staleLimitArg.split('=')[1]) : 500;
// Check GitHub rate limit
try {
const rateLimitInfo = await staleCrawler.getRateLimitStatus();
console.log(`GitHub API: ${rateLimitInfo.remaining}/${rateLimitInfo.limit} requests remaining`);
if (rateLimitInfo.remaining < 50) {
console.log('Not enough API quota for stale checking. Try again later.');
break;
}
console.log('');
} catch {
console.log('Could not check GitHub rate limit. Proceeding anyway...\n');
}
// Get skills to check (oldest scanned first)
const skillsToCheck = await skillQueries.getSkillsToStaleCheck(staleDb, staleLimit);
console.log(`Checking ${skillsToCheck.length} skills...\n`);
let staleChecked = 0;
let stillAlive = 0;
let newlyStale = 0;
let staleErrors = 0;
for (const skill of skillsToCheck) {
// Rate limit check every 50 skills
if (staleChecked > 0 && staleChecked % 50 === 0) {
try {
const midCheck = await staleCrawler.getRateLimitStatus();
if (midCheck.remaining < 20) {
console.log(`\n API quota low (${midCheck.remaining} remaining). Stopping.`);
break;
}
} catch {
// Continue if rate limit check fails
}
}
try {
const sourceFormat = skill.sourceFormat || 'skill.md';
const pattern = INSTRUCTION_FILE_PATTERNS.find(p => p.format === sourceFormat);
const filename = pattern?.filename || 'SKILL.md';
const filePath = skill.skillPath === '.' ? filename : `${skill.skillPath}/${filename}`;
const exists = await staleCrawler.checkFileExists(
skill.githubOwner,
skill.githubRepo,
filePath,
skill.branch || 'main'
);
if (exists) {
stillAlive++;
// Clear any partial stale state
if ((skill.staleCheckCount ?? 0) > 0) {
await skillQueries.clearStaleState(staleDb, skill.id);
}
} else {
await skillQueries.incrementStaleCheck(staleDb, skill.id);
newlyStale++;
console.log(` STALE: ${skill.id}`);
}
} catch {
staleErrors++;
}
staleChecked++;
// Progress every 100
if (staleChecked % 100 === 0) {
console.log(` Progress: ${staleChecked}/${skillsToCheck.length} checked`);
}
}
console.log(`\nStale check complete:`);
console.log(` Checked: ${staleChecked}`);
console.log(` Still alive: ${stillAlive}`);
console.log(` Stale detected: ${newlyStale}`);
console.log(` Errors: ${staleErrors}`);
// Show total stale count
const totalStale = await skillQueries.countStale(staleDb);
console.log(` Total stale skills in DB: ${totalStale}`);
break;
}
default: default:
console.log('Usage: pnpm crawl <command>\n'); console.log('Usage: pnpm crawl <command>\n');
@@ -944,6 +1111,7 @@ async function main() {
console.log(' sync-meili - Sync all skills from database to Meilisearch'); console.log(' sync-meili - Sync all skills from database to Meilisearch');
console.log(' recategorize - Re-categorize all skills with 23 categories (alias: link-categories)'); console.log(' recategorize - Re-categorize all skills with 23 categories (alias: link-categories)');
console.log(' process-add-requests - Process pending add requests (auto-index skills)'); console.log(' process-add-requests - Process pending add requests (auto-index skills)');
console.log(' stale-check - Check for stale skills removed from GitHub (--limit=N)');
process.exit(1); process.exit(1);
} }

View File

@@ -215,7 +215,7 @@ export class GitHubCrawler {
/** /**
* Check if a file exists in a repository * Check if a file exists in a repository
*/ */
private async checkFileExists(owner: string, repo: string, path: string, ref: string): Promise<boolean> { async checkFileExists(owner: string, repo: string, path: string, ref: string): Promise<boolean> {
try { try {
const { octokit, token } = await this.getOctokit(); const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.getContent({ const response = await octokit.repos.getContent({

View File

@@ -33,7 +33,28 @@ export async function indexSkill(
// Fetch skill content // Fetch skill content
console.log(`Fetching ${source.owner}/${source.repo}/${source.path} [${sourceFormat}]...`); console.log(`Fetching ${source.owner}/${source.repo}/${source.path} [${sourceFormat}]...`);
const content = await crawler.fetchSkillContent(source); let content;
try {
content = await crawler.fetchSkillContent(source);
} catch (error) {
const errorMsg = error instanceof Error ? error.message : String(error);
// Detect 404 (skill removed from GitHub) — increment stale check
if (errorMsg.includes('not found') || errorMsg.includes('Not Found') || errorMsg.includes('404')) {
const skillName = source.path.split('/').pop() || 'skill';
const formatSuffix = sourceFormat !== 'skill.md' ? `~${sourceFormat.replace('.', '')}` : '';
const skillId = `${source.owner}/${source.repo}/${skillName}${formatSuffix}`;
await skillQueries.incrementStaleCheck(database, skillId).catch((err) => {
console.warn(` -> Failed to increment stale check for ${skillId}:`, err);
});
console.log(` -> ${skillId}: GitHub 404 (stale check incremented)`);
return null;
}
// For non-404 errors, rethrow
throw error;
}
// Analyze the skill with format awareness // Analyze the skill with format awareness
const analysis = await analyzer.analyze(content, sourceFormat); const analysis = await analyzer.analyze(content, sourceFormat);