sync: stale skill detection, sentry filtering, claim removal, review improvements
Stale Detection: - Detect skills removed/moved from GitHub (3 consecutive 404s threshold) - Hide stale skills from browse/search, show warning banner on detail page - Serve cached files with isStale flag when GitHub returns 404 - Add stale-check crawler command for batch verification - CLI shows warning when installing stale skills from cache Sentry & Error Handling: - Filter browser extension errors and add denyUrls - Anti-inflation measures and sentinel recalibration for curation Claim & Removal: - Enhanced ClaimForm with repo-level removal support - Add repo-removal-request API endpoint with tests - Improved owner page with bilingual content Review Pipeline: - Review version and reviewer tracking in submit API - Source format filter for pending reviews - Updated review tests Other: - Updated i18n strings (en/fa) - BrowseFilters improvements - Dockerfile updates Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
19
CLAUDE.md
19
CLAUDE.md
@@ -90,10 +90,21 @@ Sanitized: `anthropics/skills/pdf` → `anthropics__skills__pdf`
|
|||||||
## Indexer
|
## Indexer
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker compose exec indexer node dist/crawl.js full # Full crawl
|
docker compose exec indexer node dist/crawl.js full # Full crawl
|
||||||
docker compose exec indexer node dist/crawl.js incremental # Last 24h
|
docker compose exec indexer node dist/crawl.js incremental # Last 24h
|
||||||
docker compose exec indexer node dist/crawl.js sync-meili # Sync to Meilisearch
|
docker compose exec indexer node dist/crawl.js sync-meili # Sync to Meilisearch
|
||||||
docker compose exec indexer node dist/crawl.js deep-scan # Scan discovered repos
|
docker compose exec indexer node dist/crawl.js deep-scan # Scan discovered repos
|
||||||
|
docker compose exec indexer node dist/crawl.js add-repo <owner/repo> # Manually add a repo to DB
|
||||||
|
docker compose exec indexer node dist/crawl.js process-add-requests # Process user-submitted add requests
|
||||||
|
```
|
||||||
|
|
||||||
|
### Database Migration (container)
|
||||||
|
```bash
|
||||||
|
# Add new columns (run when schema changes are deployed)
|
||||||
|
docker compose exec db psql -U skillhub -d skillhub -c "ALTER TABLE skills ADD COLUMN IF NOT EXISTS is_owner_claimed boolean NOT NULL DEFAULT false;"
|
||||||
|
docker compose exec db psql -U skillhub -d skillhub -c "ALTER TABLE discovered_repos ADD COLUMN IF NOT EXISTS is_blocked boolean NOT NULL DEFAULT false;"
|
||||||
|
docker compose exec db psql -U skillhub -d skillhub -c "CREATE INDEX IF NOT EXISTS idx_skills_owner_claimed ON skills(is_owner_claimed) WHERE is_owner_claimed = true;"
|
||||||
|
docker compose exec db psql -U skillhub -d skillhub -c "CREATE INDEX IF NOT EXISTS idx_discovered_repos_blocked ON discovered_repos(is_blocked) WHERE is_blocked = true;"
|
||||||
```
|
```
|
||||||
|
|
||||||
## Security
|
## Security
|
||||||
|
|||||||
@@ -98,6 +98,14 @@ export async function install(skillId: string, options: InstallOptions): Promise
|
|||||||
const cachedFiles = await getSkillFiles(skillInfo.id);
|
const cachedFiles = await getSkillFiles(skillInfo.id);
|
||||||
|
|
||||||
if (cachedFiles && cachedFiles.files.length > 0) {
|
if (cachedFiles && cachedFiles.files.length > 0) {
|
||||||
|
// Warn if skill is stale (served from cache because GitHub returned 404)
|
||||||
|
if (cachedFiles.isStale) {
|
||||||
|
spinner.warn(chalk.yellow(
|
||||||
|
'This skill may have been removed from its GitHub repository.\n' +
|
||||||
|
' Files were served from the SkillHub cache and may be outdated.'
|
||||||
|
));
|
||||||
|
spinner.start('Installing from cached files...');
|
||||||
|
}
|
||||||
// Use sourceFormat from API response if available
|
// Use sourceFormat from API response if available
|
||||||
if (cachedFiles.sourceFormat) {
|
if (cachedFiles.sourceFormat) {
|
||||||
sourceFormat = cachedFiles.sourceFormat as SourceFormat;
|
sourceFormat = cachedFiles.sourceFormat as SourceFormat;
|
||||||
|
|||||||
@@ -231,6 +231,8 @@ export interface SkillFilesResponse {
|
|||||||
files: SkillFile[];
|
files: SkillFile[];
|
||||||
fromCache: boolean;
|
fromCache: boolean;
|
||||||
cachedAt?: string;
|
cachedAt?: string;
|
||||||
|
isStale?: boolean;
|
||||||
|
staleWarning?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -45,6 +45,7 @@ ARG NEXT_PUBLIC_OPENPANEL_API_URL
|
|||||||
ARG NEXT_PUBLIC_OPENPANEL_SCRIPT_URL
|
ARG NEXT_PUBLIC_OPENPANEL_SCRIPT_URL
|
||||||
ARG NEXT_PUBLIC_UMAMI_WEBSITE_ID
|
ARG NEXT_PUBLIC_UMAMI_WEBSITE_ID
|
||||||
ARG NEXT_PUBLIC_UMAMI_URL
|
ARG NEXT_PUBLIC_UMAMI_URL
|
||||||
|
ARG NEXT_PUBLIC_IS_PRIMARY
|
||||||
|
|
||||||
# Convert ARGs to ENVs so they're available during build
|
# Convert ARGs to ENVs so they're available during build
|
||||||
ENV NEXT_PUBLIC_APP_URL=$NEXT_PUBLIC_APP_URL
|
ENV NEXT_PUBLIC_APP_URL=$NEXT_PUBLIC_APP_URL
|
||||||
@@ -56,6 +57,7 @@ ENV NEXT_PUBLIC_OPENPANEL_API_URL=$NEXT_PUBLIC_OPENPANEL_API_URL
|
|||||||
ENV NEXT_PUBLIC_OPENPANEL_SCRIPT_URL=$NEXT_PUBLIC_OPENPANEL_SCRIPT_URL
|
ENV NEXT_PUBLIC_OPENPANEL_SCRIPT_URL=$NEXT_PUBLIC_OPENPANEL_SCRIPT_URL
|
||||||
ENV NEXT_PUBLIC_UMAMI_WEBSITE_ID=$NEXT_PUBLIC_UMAMI_WEBSITE_ID
|
ENV NEXT_PUBLIC_UMAMI_WEBSITE_ID=$NEXT_PUBLIC_UMAMI_WEBSITE_ID
|
||||||
ENV NEXT_PUBLIC_UMAMI_URL=$NEXT_PUBLIC_UMAMI_URL
|
ENV NEXT_PUBLIC_UMAMI_URL=$NEXT_PUBLIC_UMAMI_URL
|
||||||
|
ENV NEXT_PUBLIC_IS_PRIMARY=$NEXT_PUBLIC_IS_PRIMARY
|
||||||
|
|
||||||
# Build the application (turbo will build dependencies first)
|
# Build the application (turbo will build dependencies first)
|
||||||
RUN pnpm turbo run build --filter=@skillhub/web
|
RUN pnpm turbo run build --filter=@skillhub/web
|
||||||
|
|||||||
@@ -186,7 +186,7 @@ export default async function BrowsePage({ params, searchParams }: BrowsePagePro
|
|||||||
|
|
||||||
const emptyStateTranslations = {
|
const emptyStateTranslations = {
|
||||||
noResults: t('noResults') || 'No skills found',
|
noResults: t('noResults') || 'No skills found',
|
||||||
noResultsWithQuery: t('noResultsWithQuery') || 'No results for "{query}"',
|
noResultsWithQuery: t('noResultsWithQuery', { query: searchParamsResolved.q || '' }),
|
||||||
tryDifferent: t('emptyState.tryDifferent') || 'Try different search terms or adjust your filters',
|
tryDifferent: t('emptyState.tryDifferent') || 'Try different search terms or adjust your filters',
|
||||||
clearFilters: t('emptyState.clearFilters') || 'Clear filters',
|
clearFilters: t('emptyState.clearFilters') || 'Clear filters',
|
||||||
browseAll: t('emptyState.browseAll') || 'Browse Featured Skills',
|
browseAll: t('emptyState.browseAll') || 'Browse Featured Skills',
|
||||||
|
|||||||
@@ -42,6 +42,7 @@ export default async function ClaimPage({
|
|||||||
tabs: {
|
tabs: {
|
||||||
remove: t('tabs.remove'),
|
remove: t('tabs.remove'),
|
||||||
add: t('tabs.add'),
|
add: t('tabs.add'),
|
||||||
|
removeRepo: t('tabs.removeRepo'),
|
||||||
},
|
},
|
||||||
form: {
|
form: {
|
||||||
skillId: t('form.skillId'),
|
skillId: t('form.skillId'),
|
||||||
@@ -78,6 +79,27 @@ export default async function ClaimPage({
|
|||||||
foundSkillsIn: t('addSuccess.foundSkillsIn'),
|
foundSkillsIn: t('addSuccess.foundSkillsIn'),
|
||||||
root: t('addSuccess.root'),
|
root: t('addSuccess.root'),
|
||||||
andMore: t.raw('addSuccess.andMore') as string,
|
andMore: t.raw('addSuccess.andMore') as string,
|
||||||
|
reEnabledTitle: t('addSuccess.reEnabledTitle'),
|
||||||
|
reEnabledDescription: t('addSuccess.reEnabledDescription'),
|
||||||
|
},
|
||||||
|
removeRepoForm: {
|
||||||
|
repoUrl: t('removeRepoForm.repoUrl'),
|
||||||
|
repoUrlPlaceholder: t('removeRepoForm.repoUrlPlaceholder'),
|
||||||
|
repoUrlHelp: t('removeRepoForm.repoUrlHelp'),
|
||||||
|
reason: t('removeRepoForm.reason'),
|
||||||
|
reasonPlaceholder: t('removeRepoForm.reasonPlaceholder'),
|
||||||
|
submit: t('removeRepoForm.submit'),
|
||||||
|
submitting: t('removeRepoForm.submitting'),
|
||||||
|
},
|
||||||
|
removeRepoSuccess: {
|
||||||
|
title: t('removeRepoSuccess.title'),
|
||||||
|
description: t.raw('removeRepoSuccess.description') as string,
|
||||||
|
descriptionZero: t('removeRepoSuccess.descriptionZero'),
|
||||||
|
},
|
||||||
|
removeRepoError: {
|
||||||
|
notOwner: t('removeRepoError.notOwner'),
|
||||||
|
invalidRepo: t('removeRepoError.invalidRepo'),
|
||||||
|
parseError: t('removeRepoError.parseError'),
|
||||||
},
|
},
|
||||||
error: {
|
error: {
|
||||||
notOwner: t('error.notOwner'),
|
notOwner: t('error.notOwner'),
|
||||||
@@ -90,6 +112,7 @@ export default async function ClaimPage({
|
|||||||
rateLimitExceeded: t('error.rateLimitExceeded'),
|
rateLimitExceeded: t('error.rateLimitExceeded'),
|
||||||
networkTimeout: t('error.networkTimeout'),
|
networkTimeout: t('error.networkTimeout'),
|
||||||
generic: t('error.generic'),
|
generic: t('error.generic'),
|
||||||
|
repoBlockedByOwner: t('error.repoBlockedByOwner'),
|
||||||
},
|
},
|
||||||
myRequests: {
|
myRequests: {
|
||||||
title: t('myRequests.title'),
|
title: t('myRequests.title'),
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import Link from 'next/link';
|
|||||||
import type { Metadata } from 'next';
|
import type { Metadata } from 'next';
|
||||||
import { getPageAlternates } from '@/lib/seo';
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
import { getOrSetCache, cacheKeys, cacheTTL } from '@/lib/cache';
|
import { getOrSetCache, cacheKeys, cacheTTL } from '@/lib/cache';
|
||||||
|
import { auth } from '@/lib/auth';
|
||||||
|
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
@@ -49,6 +50,8 @@ export default async function OwnerPage({ params, searchParams }: OwnerPageProps
|
|||||||
const activeRepo = searchParamsResolved.repo || '';
|
const activeRepo = searchParamsResolved.repo || '';
|
||||||
|
|
||||||
const db = createDb();
|
const db = createDb();
|
||||||
|
const session = await auth();
|
||||||
|
const isOwner = session?.user?.username?.toLowerCase() === username.toLowerCase();
|
||||||
|
|
||||||
// Fetch stats and repo list with caching (30 min TTL), count is dynamic per filter
|
// Fetch stats and repo list with caching (30 min TTL), count is dynamic per filter
|
||||||
const [stats, totalSkills, ownerRepos] = await Promise.all([
|
const [stats, totalSkills, ownerRepos] = await Promise.all([
|
||||||
@@ -256,16 +259,30 @@ export default async function OwnerPage({ params, searchParams }: OwnerPageProps
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Claim CTA - subtle inline hint */}
|
{/* Claim CTA */}
|
||||||
<div className="flex items-center gap-2 mb-6 text-xs text-text-muted">
|
{isOwner ? (
|
||||||
<span>{t('claimCta')}</span>
|
<div className="flex items-center gap-3 mb-6 px-4 py-3 bg-primary/5 border border-primary/20 rounded-lg">
|
||||||
<Link
|
<div className="flex-1 text-sm text-text-secondary">
|
||||||
href={`/${locale}/claim`}
|
{t('ownerManageCta')}
|
||||||
className="text-primary hover:underline font-medium whitespace-nowrap"
|
</div>
|
||||||
>
|
<Link
|
||||||
{t('claimButton')}
|
href={`/${locale}/claim`}
|
||||||
</Link>
|
className="text-sm text-primary hover:underline font-medium whitespace-nowrap"
|
||||||
</div>
|
>
|
||||||
|
{t('ownerManageButton')}
|
||||||
|
</Link>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<div className="flex items-center gap-2 mb-6 text-xs text-text-muted">
|
||||||
|
<span>{t('claimCta')}</span>
|
||||||
|
<Link
|
||||||
|
href={`/${locale}/claim`}
|
||||||
|
className="text-primary hover:underline font-medium whitespace-nowrap"
|
||||||
|
>
|
||||||
|
{t('claimButton')}
|
||||||
|
</Link>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
{/* Repos + Skills */}
|
{/* Repos + Skills */}
|
||||||
{repos.map((repo) => (
|
{repos.map((repo) => (
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ async function getStats() {
|
|||||||
const db = createDb();
|
const db = createDb();
|
||||||
|
|
||||||
// Browse-ready filter: exclude duplicates (matches browseReadyFilter in queries.ts)
|
// Browse-ready filter: exclude duplicates (matches browseReadyFilter in queries.ts)
|
||||||
const browseReady = sql`${skills.isDuplicate} = false`;
|
const browseReady = sql`${skills.isDuplicate} = false AND ${skills.isStale} = false`;
|
||||||
|
|
||||||
// Run all independent count queries in parallel
|
// Run all independent count queries in parallel
|
||||||
const [skillsResult, downloadsResult, categories, contributorsResult, totalIndexedResult] = await Promise.all([
|
const [skillsResult, downloadsResult, categories, contributorsResult, totalIndexedResult] = await Promise.all([
|
||||||
|
|||||||
@@ -339,6 +339,33 @@ export default async function SkillPage({ params }: SkillPageProps) {
|
|||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
{/* Stale Skill Warning Banner */}
|
||||||
|
{dbSkill.isStale && (
|
||||||
|
<div className="bg-warning/10 border-b border-warning/20">
|
||||||
|
<div className="container-main py-3">
|
||||||
|
<div className="flex items-start gap-3 text-sm">
|
||||||
|
<span className="text-warning text-lg flex-shrink-0">⚠</span>
|
||||||
|
<div>
|
||||||
|
<p className="text-warning-foreground dark:text-warning" dir="auto">
|
||||||
|
{isRTL
|
||||||
|
? 'این مهارت ممکن است از مخزن GitHub اصلی حذف یا جابجا شده باشد. فایلها از حافظه پنهان SkillHub ارائه میشوند و ممکن است قدیمی باشند.'
|
||||||
|
: 'This skill may have been removed or moved from its GitHub repository. Files are served from the SkillHub cache and may be outdated.'}
|
||||||
|
</p>
|
||||||
|
<a
|
||||||
|
href={`https://github.com/${skill.author}/${skill.repo}`}
|
||||||
|
target="_blank"
|
||||||
|
rel="noopener noreferrer"
|
||||||
|
className="inline-flex items-center gap-1 mt-1 text-primary-600 hover:text-primary-700 dark:text-primary-400 dark:hover:text-primary-300 font-medium"
|
||||||
|
>
|
||||||
|
{isRTL ? 'بررسی مخزن GitHub' : 'Check GitHub repository'}
|
||||||
|
<ExternalLink className="w-3 h-3" />
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
{/* Stats Bar */}
|
{/* Stats Bar */}
|
||||||
<div className="bg-surface-elevated border-b border-border">
|
<div className="bg-surface-elevated border-b border-border">
|
||||||
<div className="container-main">
|
<div className="container-main">
|
||||||
|
|||||||
@@ -49,6 +49,9 @@ export async function GET(request: NextRequest) {
|
|||||||
Math.max(parseInt(searchParams.get('owner_limit') ?? '0', 10) || 0, 0),
|
Math.max(parseInt(searchParams.get('owner_limit') ?? '0', 10) || 0, 0),
|
||||||
10
|
10
|
||||||
);
|
);
|
||||||
|
const currentReviewVersion = Math.max(
|
||||||
|
parseInt(searchParams.get('review_version') ?? '0', 10) || 0, 0
|
||||||
|
);
|
||||||
|
|
||||||
// Run counts in parallel
|
// Run counts in parallel
|
||||||
const [totalPending, reReviews] = await Promise.all([
|
const [totalPending, reReviews] = await Promise.all([
|
||||||
@@ -68,6 +71,7 @@ export async function GET(request: NextRequest) {
|
|||||||
securityPass,
|
securityPass,
|
||||||
reReviewAll: true,
|
reReviewAll: true,
|
||||||
ownerLimit,
|
ownerLimit,
|
||||||
|
currentReviewVersion,
|
||||||
}) as typeof batch;
|
}) as typeof batch;
|
||||||
batch = [...allBatch].slice(0, batchSize);
|
batch = [...allBatch].slice(0, batchSize);
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -22,6 +22,8 @@ interface ReviewItem {
|
|||||||
i18n_priority?: number;
|
i18n_priority?: number;
|
||||||
content_hash_at_review?: string;
|
content_hash_at_review?: string;
|
||||||
set_verified?: boolean;
|
set_verified?: boolean;
|
||||||
|
review_version?: number;
|
||||||
|
reviewer?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
function validateReviews(body: unknown): { reviews: ReviewItem[] } | { error: string } {
|
function validateReviews(body: unknown): { reviews: ReviewItem[] } | { error: string } {
|
||||||
@@ -60,6 +62,18 @@ function validateReviews(body: unknown): { reviews: ReviewItem[] } | { error: st
|
|||||||
return { error: `reviews[${i}].i18n_priority must be an integer 0-2` };
|
return { error: `reviews[${i}].i18n_priority must be an integer 0-2` };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Validate review_version (positive integer)
|
||||||
|
if (item.review_version !== undefined && item.review_version !== null) {
|
||||||
|
if (typeof item.review_version !== 'number' || !Number.isInteger(item.review_version) || item.review_version < 1) {
|
||||||
|
return { error: `reviews[${i}].review_version must be a positive integer` };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Validate reviewer (non-empty string, max 50 chars)
|
||||||
|
if (item.reviewer !== undefined && item.reviewer !== null) {
|
||||||
|
if (typeof item.reviewer !== 'string' || item.reviewer.length === 0 || item.reviewer.length > 50) {
|
||||||
|
return { error: `reviews[${i}].reviewer must be a non-empty string (max 50 chars)` };
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return { reviews: reviews as ReviewItem[] };
|
return { reviews: reviews as ReviewItem[] };
|
||||||
@@ -106,7 +120,7 @@ export async function POST(request: NextRequest) {
|
|||||||
// Insert review rows
|
// Insert review rows
|
||||||
const dbReviews = reviews.map((r) => ({
|
const dbReviews = reviews.map((r) => ({
|
||||||
skillId: r.skill_id,
|
skillId: r.skill_id,
|
||||||
reviewer: 'claude-code' as const,
|
reviewer: r.reviewer || 'claude-code',
|
||||||
aiScore: r.ai_score,
|
aiScore: r.ai_score,
|
||||||
instructionQuality: r.instruction_quality,
|
instructionQuality: r.instruction_quality,
|
||||||
descriptionPrecision: r.description_precision,
|
descriptionPrecision: r.description_precision,
|
||||||
@@ -119,6 +133,7 @@ export async function POST(request: NextRequest) {
|
|||||||
needsImprovement: r.needs_improvement ?? undefined,
|
needsImprovement: r.needs_improvement ?? undefined,
|
||||||
i18nPriority: r.i18n_priority,
|
i18nPriority: r.i18n_priority,
|
||||||
contentHashAtReview: r.content_hash_at_review,
|
contentHashAtReview: r.content_hash_at_review,
|
||||||
|
reviewVersion: r.review_version,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
await skillReviewQueries.createBatch(db, dbReviews);
|
await skillReviewQueries.createBatch(db, dbReviews);
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ interface SkillFile {
|
|||||||
content: string;
|
content: string;
|
||||||
size: number;
|
size: number;
|
||||||
isBinary: boolean;
|
isBinary: boolean;
|
||||||
|
fetchFailed?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface CachedFiles {
|
interface CachedFiles {
|
||||||
@@ -136,7 +137,7 @@ export async function GET(request: NextRequest) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Fetch skill folder contents from GitHub
|
// Fetch skill folder contents from GitHub
|
||||||
const files = await fetchSkillFiles(
|
const { files, fetchFailureCount } = await fetchSkillFiles(
|
||||||
githubOwner,
|
githubOwner,
|
||||||
githubRepo,
|
githubRepo,
|
||||||
skillPath,
|
skillPath,
|
||||||
@@ -146,19 +147,22 @@ export async function GET(request: NextRequest) {
|
|||||||
token
|
token
|
||||||
);
|
);
|
||||||
|
|
||||||
// === SAVE TO CACHE ===
|
// === SAVE TO CACHE (only if all files fetched successfully) ===
|
||||||
// Prepare cached files structure
|
if (fetchFailureCount > 0) {
|
||||||
const filesToCache: CachedFiles = {
|
console.warn(`[skill-files] Skipping cache for ${skillId}: ${fetchFailureCount} file(s) failed to fetch`);
|
||||||
fetchedAt: new Date().toISOString(),
|
} else {
|
||||||
commitSha: commitSha || 'unknown',
|
const filesToCache: CachedFiles = {
|
||||||
totalSize: files.reduce((sum, f) => sum + f.size, 0),
|
fetchedAt: new Date().toISOString(),
|
||||||
items: files,
|
commitSha: commitSha || 'unknown',
|
||||||
};
|
totalSize: files.reduce((sum, f) => sum + f.size, 0),
|
||||||
|
items: files.map(({ fetchFailed: _, ...rest }) => rest),
|
||||||
|
};
|
||||||
|
|
||||||
// Save to database (async, don't block response)
|
// Save to database (async, don't block response)
|
||||||
skillQueries.updateCachedFiles(db, skillId, filesToCache).catch((err) => {
|
skillQueries.updateCachedFiles(db, skillId, filesToCache).catch((err) => {
|
||||||
console.error(`[skill-files] Failed to cache files for ${skillId}:`, err);
|
console.error(`[skill-files] Failed to cache files for ${skillId}:`, err);
|
||||||
});
|
});
|
||||||
|
}
|
||||||
|
|
||||||
// Note: Download count is NOT incremented here.
|
// Note: Download count is NOT incremented here.
|
||||||
// It should only be incremented in /api/skills/install after successful download.
|
// It should only be incremented in /api/skills/install after successful download.
|
||||||
@@ -180,6 +184,7 @@ export async function GET(request: NextRequest) {
|
|||||||
downloadUrl: item.isBinary ? `https://raw.githubusercontent.com/${githubOwner}/${githubRepo}/${branch || 'main'}/${skillPath}/${item.path}` : undefined,
|
downloadUrl: item.isBinary ? `https://raw.githubusercontent.com/${githubOwner}/${githubRepo}/${branch || 'main'}/${skillPath}/${item.path}` : undefined,
|
||||||
})),
|
})),
|
||||||
fromCache: false,
|
fromCache: false,
|
||||||
|
...(fetchFailureCount > 0 ? { fetchFailures: fetchFailureCount } : {}),
|
||||||
}, {
|
}, {
|
||||||
headers: createRateLimitHeaders(rateLimitResult),
|
headers: createRateLimitHeaders(rateLimitResult),
|
||||||
});
|
});
|
||||||
@@ -204,6 +209,44 @@ export async function GET(request: NextRequest) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (errorMessage.includes('404')) {
|
if (errorMessage.includes('404')) {
|
||||||
|
// Increment stale check (non-blocking) — after 3 consecutive 404s, skill is marked stale
|
||||||
|
const skillId404 = request.nextUrl.searchParams.get('id');
|
||||||
|
if (skillId404) {
|
||||||
|
skillQueries.incrementStaleCheck(db, skillId404).catch((err) => {
|
||||||
|
console.error(`[skill-files] Failed to increment stale check for ${skillId404}:`, err);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// If we have cached files in DB, serve them with stale warning instead of 404
|
||||||
|
if (skillId404) {
|
||||||
|
const skill404 = await skillQueries.getById(db, skillId404);
|
||||||
|
if (skill404?.cachedFiles) {
|
||||||
|
const cached = skill404.cachedFiles as CachedFiles;
|
||||||
|
return NextResponse.json({
|
||||||
|
skillId: skillId404,
|
||||||
|
githubOwner: skill404.githubOwner,
|
||||||
|
githubRepo: skill404.githubRepo,
|
||||||
|
skillPath: skill404.skillPath,
|
||||||
|
branch: skill404.branch || 'main',
|
||||||
|
sourceFormat: skill404.sourceFormat || 'skill.md',
|
||||||
|
files: cached.items.map(item => ({
|
||||||
|
name: item.name,
|
||||||
|
path: item.path,
|
||||||
|
type: 'file' as const,
|
||||||
|
size: item.size,
|
||||||
|
content: item.isBinary ? undefined : item.content,
|
||||||
|
downloadUrl: item.isBinary ? `https://raw.githubusercontent.com/${skill404.githubOwner}/${skill404.githubRepo}/${skill404.branch || 'main'}/${skill404.skillPath}/${item.path}` : undefined,
|
||||||
|
})),
|
||||||
|
fromCache: true,
|
||||||
|
cachedAt: cached.fetchedAt,
|
||||||
|
isStale: true,
|
||||||
|
staleWarning: 'This skill may have been removed from GitHub. Files served from cache.',
|
||||||
|
}, {
|
||||||
|
headers: createRateLimitHeaders(rateLimitResult),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{ error: 'Skill files not found on GitHub', code: 'GITHUB_NOT_FOUND' },
|
{ error: 'Skill files not found on GitHub', code: 'GITHUB_NOT_FOUND' },
|
||||||
{ status: 404 }
|
{ status: 404 }
|
||||||
@@ -314,7 +357,7 @@ async function fetchSkillFiles(
|
|||||||
depth: number = 0,
|
depth: number = 0,
|
||||||
headers: Record<string, string>,
|
headers: Record<string, string>,
|
||||||
token: string | null
|
token: string | null
|
||||||
): Promise<SkillFile[]> {
|
): Promise<{ files: SkillFile[]; fetchFailureCount: number }> {
|
||||||
// Phase 1: Collect all file entries from directory tree
|
// Phase 1: Collect all file entries from directory tree
|
||||||
const entries = await collectFileEntries(owner, repo, path, ref, depth, headers, token);
|
const entries = await collectFileEntries(owner, repo, path, ref, depth, headers, token);
|
||||||
|
|
||||||
@@ -326,6 +369,7 @@ async function fetchSkillFiles(
|
|||||||
|
|
||||||
// Phase 2: Fetch file contents in parallel
|
// Phase 2: Fetch file contents in parallel
|
||||||
let totalSize = 0;
|
let totalSize = 0;
|
||||||
|
let fetchFailureCount = 0;
|
||||||
const tasks = entries.map((entry) => async (): Promise<SkillFile> => {
|
const tasks = entries.map((entry) => async (): Promise<SkillFile> => {
|
||||||
const { item, relativePath } = entry;
|
const { item, relativePath } = entry;
|
||||||
const isBinary = !isTextFile(item.name);
|
const isBinary = !isTextFile(item.name);
|
||||||
@@ -340,15 +384,18 @@ async function fetchSkillFiles(
|
|||||||
content,
|
content,
|
||||||
size: item.size,
|
size: item.size,
|
||||||
isBinary: false,
|
isBinary: false,
|
||||||
|
fetchFailed: false,
|
||||||
};
|
};
|
||||||
} catch {
|
} catch {
|
||||||
// Content fetch failed — return as binary with download URL
|
// Content fetch failed — mark as failed so we skip caching
|
||||||
|
fetchFailureCount++;
|
||||||
return {
|
return {
|
||||||
name: item.name,
|
name: item.name,
|
||||||
path: relativePath,
|
path: relativePath,
|
||||||
content: '',
|
content: '',
|
||||||
size: item.size,
|
size: item.size,
|
||||||
isBinary: true,
|
isBinary: true,
|
||||||
|
fetchFailed: true,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -360,10 +407,12 @@ async function fetchSkillFiles(
|
|||||||
content: '',
|
content: '',
|
||||||
size: item.size,
|
size: item.size,
|
||||||
isBinary: true,
|
isBinary: true,
|
||||||
|
fetchFailed: false,
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
return parallelLimit(tasks, PARALLEL_FETCH_LIMIT);
|
const files = await parallelLimit(tasks, PARALLEL_FETCH_LIMIT);
|
||||||
|
return { files, fetchFailureCount };
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
229
apps/web/app/api/skills/add-request/route.test.ts
Normal file
229
apps/web/app/api/skills/add-request/route.test.ts
Normal file
@@ -0,0 +1,229 @@
|
|||||||
|
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||||
|
import { NextRequest } from 'next/server';
|
||||||
|
|
||||||
|
// Mock auth
|
||||||
|
const mockAuth = vi.fn();
|
||||||
|
vi.mock('@/lib/auth', () => ({
|
||||||
|
auth: () => mockAuth(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Mock sanitize
|
||||||
|
vi.mock('@/lib/sanitize', () => ({
|
||||||
|
sanitizeReason: (r: string) => r,
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Mock email
|
||||||
|
vi.mock('@/lib/email', () => ({
|
||||||
|
sendClaimSubmittedEmail: vi.fn().mockResolvedValue(undefined),
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Mock db
|
||||||
|
vi.mock('@skillhub/db', () => ({
|
||||||
|
createDb: vi.fn(() => ({})),
|
||||||
|
userQueries: {
|
||||||
|
getByGithubId: vi.fn(),
|
||||||
|
upsertFromGithub: vi.fn(),
|
||||||
|
},
|
||||||
|
skillQueries: {
|
||||||
|
unblockByRepo: vi.fn(),
|
||||||
|
},
|
||||||
|
addRequestQueries: {
|
||||||
|
hasPendingRequest: vi.fn(),
|
||||||
|
create: vi.fn(),
|
||||||
|
updateStatus: vi.fn(),
|
||||||
|
},
|
||||||
|
discoveredRepoQueries: {
|
||||||
|
getById: vi.fn(),
|
||||||
|
upsert: vi.fn(),
|
||||||
|
unblockRepo: vi.fn(),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
import { POST, GET } from './route';
|
||||||
|
import { userQueries, skillQueries, addRequestQueries, discoveredRepoQueries } from '@skillhub/db';
|
||||||
|
|
||||||
|
function createMockUser(overrides: Partial<{ id: string; githubId: string; email: string }> = {}) {
|
||||||
|
return {
|
||||||
|
id: 'user-123',
|
||||||
|
githubId: 'gh-12345',
|
||||||
|
username: 'testuser',
|
||||||
|
email: 'test@example.com',
|
||||||
|
preferredLocale: 'en',
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function createRequest(body: unknown) {
|
||||||
|
return new NextRequest('http://localhost:3000/api/skills/add-request', {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify(body),
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('/api/skills/add-request', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.clearAllMocks();
|
||||||
|
// Disable real fetch by default
|
||||||
|
vi.stubGlobal('fetch', vi.fn());
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('GET', () => {
|
||||||
|
it('should return 401 when not authenticated', async () => {
|
||||||
|
mockAuth.mockResolvedValue(null);
|
||||||
|
|
||||||
|
const response = await GET();
|
||||||
|
const data = await response.json();
|
||||||
|
|
||||||
|
expect(response.status).toBe(401);
|
||||||
|
expect(data.error).toBeDefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return empty requests for unknown user', async () => {
|
||||||
|
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345' } });
|
||||||
|
vi.mocked(userQueries.getByGithubId).mockResolvedValue(null as any);
|
||||||
|
|
||||||
|
const response = await GET();
|
||||||
|
const data = await response.json();
|
||||||
|
|
||||||
|
expect(response.status).toBe(200);
|
||||||
|
expect(data.requests).toEqual([]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('POST', () => {
|
||||||
|
it('should return 401 when not authenticated', async () => {
|
||||||
|
mockAuth.mockResolvedValue(null);
|
||||||
|
|
||||||
|
const response = await POST(createRequest({ repositoryUrl: 'https://github.com/owner/repo' }));
|
||||||
|
const data = await response.json();
|
||||||
|
|
||||||
|
expect(response.status).toBe(401);
|
||||||
|
expect(data.code).toBe('AUTH_REQUIRED');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return 400 for missing repositoryUrl', async () => {
|
||||||
|
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345', username: 'testuser' } });
|
||||||
|
|
||||||
|
const response = await POST(createRequest({}));
|
||||||
|
const data = await response.json();
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
expect(data.code).toBe('INVALID_INPUT');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return 400 for invalid GitHub URL', async () => {
|
||||||
|
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345', username: 'testuser' } });
|
||||||
|
const mockUser = createMockUser();
|
||||||
|
vi.mocked(userQueries.getByGithubId).mockResolvedValue(mockUser as any);
|
||||||
|
|
||||||
|
// gitlab.com does not contain the substring 'github.com'
|
||||||
|
const response = await POST(createRequest({ repositoryUrl: 'https://gitlab.com/owner/repo' }));
|
||||||
|
const data = await response.json();
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
expect(data.code).toBe('INVALID_URL');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return 403 REPO_BLOCKED_BY_OWNER when non-owner tries to add blocked repo', async () => {
|
||||||
|
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345', username: 'non-owner' } });
|
||||||
|
const mockUser = createMockUser({ githubId: 'gh-12345' });
|
||||||
|
vi.mocked(userQueries.getByGithubId).mockResolvedValue(mockUser as any);
|
||||||
|
vi.mocked(discoveredRepoQueries.getById).mockResolvedValue({
|
||||||
|
id: 'rawveg/skillsforge-marketplace',
|
||||||
|
isBlocked: true,
|
||||||
|
} as any);
|
||||||
|
// GitHub API says the owner is 'rawveg', not 'non-owner'
|
||||||
|
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
|
||||||
|
ok: true,
|
||||||
|
json: async () => ({ owner: { login: 'rawveg' }, default_branch: 'main', private: false }),
|
||||||
|
} as any));
|
||||||
|
|
||||||
|
const response = await POST(createRequest({ repositoryUrl: 'https://github.com/rawveg/skillsforge-marketplace' }));
|
||||||
|
const data = await response.json();
|
||||||
|
|
||||||
|
expect(response.status).toBe(403);
|
||||||
|
expect(data.code).toBe('REPO_BLOCKED_BY_OWNER');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should re-enable repo when owner re-adds their blocked repo', async () => {
|
||||||
|
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345', username: 'rawveg' } });
|
||||||
|
const mockUser = createMockUser({ githubId: 'gh-12345' });
|
||||||
|
vi.mocked(userQueries.getByGithubId).mockResolvedValue(mockUser as any);
|
||||||
|
vi.mocked(discoveredRepoQueries.getById).mockResolvedValue({
|
||||||
|
id: 'rawveg/skillsforge-marketplace',
|
||||||
|
isBlocked: true,
|
||||||
|
} as any);
|
||||||
|
vi.mocked(skillQueries.unblockByRepo).mockResolvedValue(undefined);
|
||||||
|
vi.mocked(discoveredRepoQueries.unblockRepo).mockResolvedValue(undefined);
|
||||||
|
vi.mocked(addRequestQueries.hasPendingRequest).mockResolvedValue(false as any);
|
||||||
|
vi.mocked(addRequestQueries.create).mockResolvedValue('req-123' as any);
|
||||||
|
vi.mocked(addRequestQueries.updateStatus).mockResolvedValue(undefined);
|
||||||
|
vi.mocked(discoveredRepoQueries.upsert).mockResolvedValue(undefined as any);
|
||||||
|
|
||||||
|
// First fetch: ownership check (inside blocked-repo logic)
|
||||||
|
// Second fetch: repo validation (validateGitHubRepo → repoResponse)
|
||||||
|
// Third fetch: tree scan (findSkillMdFiles)
|
||||||
|
vi.stubGlobal('fetch', vi.fn()
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
ok: true,
|
||||||
|
json: async () => ({ owner: { login: 'rawveg' }, default_branch: 'main', private: false }),
|
||||||
|
} as any)
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
ok: true,
|
||||||
|
json: async () => ({ owner: { login: 'rawveg' }, default_branch: 'main', private: false }),
|
||||||
|
} as any)
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
ok: true,
|
||||||
|
json: async () => ({
|
||||||
|
tree: [
|
||||||
|
{ path: 'SKILL.md', type: 'blob' },
|
||||||
|
],
|
||||||
|
truncated: false,
|
||||||
|
}),
|
||||||
|
} as any)
|
||||||
|
);
|
||||||
|
|
||||||
|
const response = await POST(createRequest({ repositoryUrl: 'https://github.com/rawveg/skillsforge-marketplace' }));
|
||||||
|
const data = await response.json();
|
||||||
|
|
||||||
|
expect(response.status).toBe(200);
|
||||||
|
expect(data.success).toBe(true);
|
||||||
|
expect(data.reEnabled).toBe(true);
|
||||||
|
expect(skillQueries.unblockByRepo).toHaveBeenCalledWith(expect.anything(), 'rawveg', 'skillsforge-marketplace');
|
||||||
|
expect(discoveredRepoQueries.unblockRepo).toHaveBeenCalledWith(expect.anything(), 'rawveg/skillsforge-marketplace');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should proceed normally when repo is not blocked (Case C)', async () => {
|
||||||
|
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345', username: 'newuser' } });
|
||||||
|
const mockUser = createMockUser();
|
||||||
|
vi.mocked(userQueries.getByGithubId).mockResolvedValue(mockUser as any);
|
||||||
|
vi.mocked(discoveredRepoQueries.getById).mockResolvedValue(null as any);
|
||||||
|
vi.mocked(addRequestQueries.hasPendingRequest).mockResolvedValue(false as any);
|
||||||
|
vi.mocked(addRequestQueries.create).mockResolvedValue('req-456' as any);
|
||||||
|
vi.mocked(addRequestQueries.updateStatus).mockResolvedValue(undefined);
|
||||||
|
vi.mocked(discoveredRepoQueries.upsert).mockResolvedValue(undefined as any);
|
||||||
|
|
||||||
|
vi.stubGlobal('fetch', vi.fn()
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
ok: true,
|
||||||
|
json: async () => ({ owner: { login: 'someowner' }, default_branch: 'main', private: false }),
|
||||||
|
} as any)
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
ok: true,
|
||||||
|
json: async () => ({
|
||||||
|
tree: [{ path: 'SKILL.md', type: 'blob' }],
|
||||||
|
truncated: false,
|
||||||
|
}),
|
||||||
|
} as any)
|
||||||
|
);
|
||||||
|
|
||||||
|
const response = await POST(createRequest({ repositoryUrl: 'https://github.com/someowner/newrepo' }));
|
||||||
|
const data = await response.json();
|
||||||
|
|
||||||
|
expect(response.status).toBe(200);
|
||||||
|
expect(data.success).toBe(true);
|
||||||
|
expect(data.reEnabled).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
import type { NextRequest } from 'next/server';
|
import type { NextRequest } from 'next/server';
|
||||||
import { NextResponse } from 'next/server';
|
import { NextResponse } from 'next/server';
|
||||||
import { auth } from '@/lib/auth';
|
import { auth } from '@/lib/auth';
|
||||||
import { createDb, addRequestQueries, discoveredRepoQueries, userQueries } from '@skillhub/db';
|
import { createDb, addRequestQueries, discoveredRepoQueries, skillQueries, userQueries } from '@skillhub/db';
|
||||||
import { sanitizeReason } from '@/lib/sanitize';
|
import { sanitizeReason } from '@/lib/sanitize';
|
||||||
import { sendClaimSubmittedEmail } from '@/lib/email';
|
import { sendClaimSubmittedEmail } from '@/lib/email';
|
||||||
|
|
||||||
@@ -337,6 +337,52 @@ export async function POST(request: NextRequest) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Check if repo was previously blocked by its owner
|
||||||
|
let reEnabled = false;
|
||||||
|
const existingRepo = await discoveredRepoQueries.getById(db, `${parsed.owner}/${parsed.repo}`);
|
||||||
|
if (existingRepo?.isBlocked) {
|
||||||
|
// Verify whether the current requester is the repo owner
|
||||||
|
let requesterIsOwner = false;
|
||||||
|
try {
|
||||||
|
const ownerCheckRes = await fetch(
|
||||||
|
`https://api.github.com/repos/${parsed.owner}/${parsed.repo}`,
|
||||||
|
{
|
||||||
|
headers: {
|
||||||
|
Accept: 'application/vnd.github.v3+json',
|
||||||
|
'User-Agent': 'SkillHub',
|
||||||
|
...(process.env.GITHUB_TOKEN && {
|
||||||
|
Authorization: `Bearer ${process.env.GITHUB_TOKEN}`,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
signal: AbortSignal.timeout(10000),
|
||||||
|
}
|
||||||
|
);
|
||||||
|
if (ownerCheckRes.ok) {
|
||||||
|
const repoData = await ownerCheckRes.json() as { owner?: { login?: string } };
|
||||||
|
requesterIsOwner =
|
||||||
|
repoData.owner?.login?.toLowerCase() === session.user.username.toLowerCase();
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// If we can't reach GitHub, treat as non-owner (safe default)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!requesterIsOwner) {
|
||||||
|
// Case B: repo blocked by its owner, non-owner trying to re-add
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
error: 'This repository was removed by its owner and cannot be re-added.',
|
||||||
|
code: 'REPO_BLOCKED_BY_OWNER',
|
||||||
|
},
|
||||||
|
{ status: 403 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Case A: owner is re-enabling their previously blocked repo
|
||||||
|
await skillQueries.unblockByRepo(db, parsed.owner, parsed.repo);
|
||||||
|
await discoveredRepoQueries.unblockRepo(db, `${parsed.owner}/${parsed.repo}`);
|
||||||
|
reEnabled = true;
|
||||||
|
}
|
||||||
|
|
||||||
// Check if user already has a pending request for this repository + path combination
|
// Check if user already has a pending request for this repository + path combination
|
||||||
const hasPending = await addRequestQueries.hasPendingRequest(
|
const hasPending = await addRequestQueries.hasPendingRequest(
|
||||||
db,
|
db,
|
||||||
@@ -438,6 +484,7 @@ export async function POST(request: NextRequest) {
|
|||||||
skillCount: validation.skillPaths.length,
|
skillCount: validation.skillPaths.length,
|
||||||
skillPaths: validation.skillPaths,
|
skillPaths: validation.skillPaths,
|
||||||
message,
|
message,
|
||||||
|
...(reEnabled && { reEnabled: true }),
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error creating add request:', error);
|
console.error('Error creating add request:', error);
|
||||||
|
|||||||
163
apps/web/app/api/skills/repo-removal-request/route.test.ts
Normal file
163
apps/web/app/api/skills/repo-removal-request/route.test.ts
Normal file
@@ -0,0 +1,163 @@
|
|||||||
|
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||||
|
import { NextRequest } from 'next/server';
|
||||||
|
|
||||||
|
// Mock auth
|
||||||
|
const mockAuth = vi.fn();
|
||||||
|
vi.mock('@/lib/auth', () => ({
|
||||||
|
auth: () => mockAuth(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Mock sanitize
|
||||||
|
vi.mock('@/lib/sanitize', () => ({
|
||||||
|
sanitizeReason: (r: string) => r,
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Mock db
|
||||||
|
vi.mock('@skillhub/db', () => ({
|
||||||
|
createDb: vi.fn(() => ({})),
|
||||||
|
userQueries: {
|
||||||
|
getByGithubId: vi.fn(),
|
||||||
|
upsertFromGithub: vi.fn(),
|
||||||
|
},
|
||||||
|
skillQueries: {
|
||||||
|
countByRepo: vi.fn(),
|
||||||
|
blockByRepo: vi.fn(),
|
||||||
|
},
|
||||||
|
discoveredRepoQueries: {
|
||||||
|
blockRepo: vi.fn(),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
import { POST } from './route';
|
||||||
|
import { userQueries, skillQueries, discoveredRepoQueries } from '@skillhub/db';
|
||||||
|
|
||||||
|
function createMockUser(overrides: Partial<{ id: string; githubId: string }> = {}) {
|
||||||
|
return {
|
||||||
|
id: 'user-123',
|
||||||
|
githubId: 'gh-12345',
|
||||||
|
username: 'rawveg',
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function createRequest(body: unknown) {
|
||||||
|
return new NextRequest('http://localhost:3000/api/skills/repo-removal-request', {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify(body),
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('/api/skills/repo-removal-request', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.clearAllMocks();
|
||||||
|
vi.stubGlobal('fetch', vi.fn());
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('POST', () => {
|
||||||
|
it('should return 401 when not authenticated', async () => {
|
||||||
|
mockAuth.mockResolvedValue(null);
|
||||||
|
|
||||||
|
const response = await POST(createRequest({ repoUrl: 'rawveg/skillsforge-marketplace' }));
|
||||||
|
const data = await response.json();
|
||||||
|
|
||||||
|
expect(response.status).toBe(401);
|
||||||
|
expect(data.code).toBe('AUTH_REQUIRED');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return 400 for missing repoUrl', async () => {
|
||||||
|
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345', username: 'rawveg' } });
|
||||||
|
|
||||||
|
const response = await POST(createRequest({}));
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return 400 PARSE_ERROR for invalid repo format', async () => {
|
||||||
|
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345', username: 'rawveg' } });
|
||||||
|
const mockUser = createMockUser();
|
||||||
|
vi.mocked(userQueries.getByGithubId).mockResolvedValue(mockUser as any);
|
||||||
|
|
||||||
|
const response = await POST(createRequest({ repoUrl: 'not-a-valid-format' }));
|
||||||
|
const data = await response.json();
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
expect(data.code).toBe('PARSE_ERROR');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return 404 INVALID_REPO when repo not found on GitHub', async () => {
|
||||||
|
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345', username: 'rawveg' } });
|
||||||
|
const mockUser = createMockUser();
|
||||||
|
vi.mocked(userQueries.getByGithubId).mockResolvedValue(mockUser as any);
|
||||||
|
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
|
||||||
|
ok: false,
|
||||||
|
status: 404,
|
||||||
|
} as any));
|
||||||
|
|
||||||
|
const response = await POST(createRequest({ repoUrl: 'rawveg/skillsforge-marketplace' }));
|
||||||
|
const data = await response.json();
|
||||||
|
|
||||||
|
expect(response.status).toBe(404);
|
||||||
|
expect(data.code).toBe('INVALID_REPO');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return 403 NOT_OWNER when requester is not repo owner', async () => {
|
||||||
|
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345', username: 'other-user' } });
|
||||||
|
const mockUser = createMockUser({ githubId: 'gh-12345' });
|
||||||
|
vi.mocked(userQueries.getByGithubId).mockResolvedValue(mockUser as any);
|
||||||
|
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
|
||||||
|
ok: true,
|
||||||
|
json: async () => ({ owner: { login: 'rawveg' } }),
|
||||||
|
} as any));
|
||||||
|
|
||||||
|
const response = await POST(createRequest({ repoUrl: 'rawveg/skillsforge-marketplace' }));
|
||||||
|
const data = await response.json();
|
||||||
|
|
||||||
|
expect(response.status).toBe(403);
|
||||||
|
expect(data.code).toBe('NOT_OWNER');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should block all skills and return success when owner removes repo', async () => {
|
||||||
|
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345', username: 'rawveg' } });
|
||||||
|
const mockUser = createMockUser();
|
||||||
|
vi.mocked(userQueries.getByGithubId).mockResolvedValue(mockUser as any);
|
||||||
|
vi.mocked(skillQueries.countByRepo).mockResolvedValue(16 as any);
|
||||||
|
vi.mocked(skillQueries.blockByRepo).mockResolvedValue(undefined);
|
||||||
|
vi.mocked(discoveredRepoQueries.blockRepo).mockResolvedValue(undefined);
|
||||||
|
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
|
||||||
|
ok: true,
|
||||||
|
json: async () => ({ owner: { login: 'rawveg' } }),
|
||||||
|
} as any));
|
||||||
|
|
||||||
|
const response = await POST(createRequest({ repoUrl: 'rawveg/skillsforge-marketplace', reason: 'I want my skills removed' }));
|
||||||
|
const data = await response.json();
|
||||||
|
|
||||||
|
expect(response.status).toBe(200);
|
||||||
|
expect(data.success).toBe(true);
|
||||||
|
expect(data.blockedCount).toBe(16);
|
||||||
|
expect(data.repo).toBe('rawveg/skillsforge-marketplace');
|
||||||
|
expect(skillQueries.blockByRepo).toHaveBeenCalledWith(expect.anything(), 'rawveg', 'skillsforge-marketplace');
|
||||||
|
expect(discoveredRepoQueries.blockRepo).toHaveBeenCalledWith(expect.anything(), 'rawveg/skillsforge-marketplace');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should accept full GitHub URL format', async () => {
|
||||||
|
mockAuth.mockResolvedValue({ user: { githubId: 'gh-12345', username: 'rawveg' } });
|
||||||
|
const mockUser = createMockUser();
|
||||||
|
vi.mocked(userQueries.getByGithubId).mockResolvedValue(mockUser as any);
|
||||||
|
vi.mocked(skillQueries.countByRepo).mockResolvedValue(0 as any);
|
||||||
|
vi.mocked(skillQueries.blockByRepo).mockResolvedValue(undefined);
|
||||||
|
vi.mocked(discoveredRepoQueries.blockRepo).mockResolvedValue(undefined);
|
||||||
|
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
|
||||||
|
ok: true,
|
||||||
|
json: async () => ({ owner: { login: 'rawveg' } }),
|
||||||
|
} as any));
|
||||||
|
|
||||||
|
const response = await POST(createRequest({ repoUrl: 'https://github.com/rawveg/skillsforge-marketplace' }));
|
||||||
|
const data = await response.json();
|
||||||
|
|
||||||
|
expect(response.status).toBe(200);
|
||||||
|
expect(data.success).toBe(true);
|
||||||
|
expect(data.repo).toBe('rawveg/skillsforge-marketplace');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
158
apps/web/app/api/skills/repo-removal-request/route.ts
Normal file
158
apps/web/app/api/skills/repo-removal-request/route.ts
Normal file
@@ -0,0 +1,158 @@
|
|||||||
|
import type { NextRequest } from 'next/server';
|
||||||
|
import { NextResponse } from 'next/server';
|
||||||
|
import { auth } from '@/lib/auth';
|
||||||
|
import { createDb, skillQueries, discoveredRepoQueries, userQueries } from '@skillhub/db';
|
||||||
|
import { sanitizeReason } from '@/lib/sanitize';
|
||||||
|
|
||||||
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
|
const db = createDb();
|
||||||
|
|
||||||
|
function parseOwnerRepo(input: string): { owner: string; repo: string } | null {
|
||||||
|
try {
|
||||||
|
if (input.startsWith('https://') || input.startsWith('http://')) {
|
||||||
|
const url = new URL(input);
|
||||||
|
if (!url.hostname.includes('github.com')) return null;
|
||||||
|
const parts = url.pathname.split('/').filter(Boolean);
|
||||||
|
if (parts.length < 2) return null;
|
||||||
|
return { owner: parts[0], repo: parts[1] };
|
||||||
|
}
|
||||||
|
const parts = input.split('/').filter(Boolean);
|
||||||
|
if (parts.length < 2) return null;
|
||||||
|
return { owner: parts[0], repo: parts[1] };
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* POST /api/skills/repo-removal-request
|
||||||
|
* Block all skills from a GitHub repository at once.
|
||||||
|
* Verifies the authenticated user owns the repo, then blocks all skills and
|
||||||
|
* prevents the repo from being re-indexed.
|
||||||
|
*/
|
||||||
|
export async function POST(request: NextRequest) {
|
||||||
|
try {
|
||||||
|
const session = await auth();
|
||||||
|
|
||||||
|
if (!session?.user?.githubId || !session.user.username) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: 'Authentication required', code: 'AUTH_REQUIRED' },
|
||||||
|
{ status: 401 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const body = await request.json();
|
||||||
|
const { repoUrl, reason } = body;
|
||||||
|
|
||||||
|
if (!repoUrl) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: 'repoUrl is required', code: 'INVALID_INPUT' },
|
||||||
|
{ status: 400 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const parsed = parseOwnerRepo(String(repoUrl).trim());
|
||||||
|
if (!parsed) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: 'Invalid format. Use https://github.com/owner/repo or owner/repo', code: 'PARSE_ERROR' },
|
||||||
|
{ status: 400 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const { owner, repo } = parsed;
|
||||||
|
const username = session.user.username;
|
||||||
|
|
||||||
|
// Get or create user
|
||||||
|
let dbUser = await userQueries.getByGithubId(db, session.user.githubId);
|
||||||
|
if (!dbUser) {
|
||||||
|
dbUser = await userQueries.upsertFromGithub(db, {
|
||||||
|
githubId: session.user.githubId,
|
||||||
|
username: session.user.username,
|
||||||
|
displayName: session.user.name || undefined,
|
||||||
|
email: session.user.email || undefined,
|
||||||
|
avatarUrl: session.user.image || undefined,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (!dbUser) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: 'Failed to create user record', code: 'USER_CREATE_FAILED' },
|
||||||
|
{ status: 500 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify repo ownership via GitHub API
|
||||||
|
let isOwner = false;
|
||||||
|
let githubError: string | null = null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const repoResponse = await fetch(
|
||||||
|
`https://api.github.com/repos/${owner}/${repo}`,
|
||||||
|
{
|
||||||
|
headers: {
|
||||||
|
Accept: 'application/vnd.github.v3+json',
|
||||||
|
'User-Agent': 'SkillHub',
|
||||||
|
},
|
||||||
|
signal: AbortSignal.timeout(10000),
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
if (repoResponse.ok) {
|
||||||
|
const repoData = await repoResponse.json();
|
||||||
|
if (repoData.owner?.login?.toLowerCase() === username.toLowerCase()) {
|
||||||
|
isOwner = true;
|
||||||
|
}
|
||||||
|
} else if (repoResponse.status === 404) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: 'Repository not found on GitHub', code: 'INVALID_REPO' },
|
||||||
|
{ status: 404 }
|
||||||
|
);
|
||||||
|
} else if (repoResponse.status === 403) {
|
||||||
|
githubError = 'GitHub API rate limit exceeded. Please try again later.';
|
||||||
|
}
|
||||||
|
} catch (fetchError) {
|
||||||
|
console.error('GitHub API fetch error:', fetchError);
|
||||||
|
githubError = 'Failed to verify repository ownership';
|
||||||
|
}
|
||||||
|
|
||||||
|
if (githubError) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: githubError, code: 'GITHUB_ERROR' },
|
||||||
|
{ status: 502 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!isOwner) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: 'You are not the owner of this repository', code: 'NOT_OWNER' },
|
||||||
|
{ status: 403 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count skills before blocking (for response message)
|
||||||
|
const blockedCount = await skillQueries.countByRepo(db, owner, repo);
|
||||||
|
|
||||||
|
// Block all skills from the repo
|
||||||
|
await skillQueries.blockByRepo(db, owner, repo);
|
||||||
|
|
||||||
|
// Block the discovered_repo entry to prevent re-indexing
|
||||||
|
await discoveredRepoQueries.blockRepo(db, `${owner}/${repo}`);
|
||||||
|
|
||||||
|
const sanitizedReason = reason ? sanitizeReason(String(reason)) : undefined;
|
||||||
|
console.log(
|
||||||
|
`[RepoRemoval] ${username} removed ${owner}/${repo} (${blockedCount} skills blocked)${sanitizedReason ? ` — ${sanitizedReason}` : ''}`
|
||||||
|
);
|
||||||
|
|
||||||
|
return NextResponse.json({
|
||||||
|
success: true,
|
||||||
|
blockedCount,
|
||||||
|
repo: `${owner}/${repo}`,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error processing repo removal request:', error);
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: 'Internal server error', code: 'SERVER_ERROR' },
|
||||||
|
{ status: 500 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -35,7 +35,7 @@ export async function GET(request: NextRequest) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Browse-ready filter: exclude duplicates (matches browseReadyFilter in queries.ts)
|
// Browse-ready filter: exclude duplicates (matches browseReadyFilter in queries.ts)
|
||||||
const browseReady = sql`${skills.isDuplicate} = false`;
|
const browseReady = sql`${skills.isDuplicate} = false AND ${skills.isStale} = false`;
|
||||||
|
|
||||||
// Browse-ready skill stats (skills count + contributors)
|
// Browse-ready skill stats (skills count + contributors)
|
||||||
const statsResult = await db
|
const statsResult = await db
|
||||||
|
|||||||
@@ -470,7 +470,7 @@ export function EmptyState({ query, hasFilters, locale = 'en', translations }: E
|
|||||||
<Sparkles className="w-10 h-10 text-text-muted" />
|
<Sparkles className="w-10 h-10 text-text-muted" />
|
||||||
</div>
|
</div>
|
||||||
<h3 className="text-xl font-semibold text-text-primary mb-2">
|
<h3 className="text-xl font-semibold text-text-primary mb-2">
|
||||||
{query ? translations.noResultsWithQuery.replace('{query}', query) : translations.noResults}
|
{query ? translations.noResultsWithQuery : translations.noResults}
|
||||||
</h3>
|
</h3>
|
||||||
<p className="text-text-secondary mb-6 max-w-md mx-auto">
|
<p className="text-text-secondary mb-6 max-w-md mx-auto">
|
||||||
{translations.tryDifferent}
|
{translations.tryDifferent}
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
import { useState, useEffect, useCallback } from 'react';
|
import { useState, useEffect, useCallback } from 'react';
|
||||||
import { useSession, signIn } from 'next-auth/react';
|
import { useSession, signIn } from 'next-auth/react';
|
||||||
import { Loader2, CheckCircle, AlertCircle, Github, Clock, Plus, Minus, ChevronDown, ChevronUp, ExternalLink } from 'lucide-react';
|
import { Loader2, CheckCircle, AlertCircle, Github, Clock, Plus, Minus, ChevronDown, ChevronUp, ExternalLink, Trash2 } from 'lucide-react';
|
||||||
import { fetchWithCsrf } from '@/lib/csrf-client';
|
import { fetchWithCsrf } from '@/lib/csrf-client';
|
||||||
|
|
||||||
const isMirror = process.env.NEXT_PUBLIC_IS_PRIMARY === 'false';
|
const isMirror = process.env.NEXT_PUBLIC_IS_PRIMARY === 'false';
|
||||||
@@ -33,6 +33,26 @@ interface ClaimFormProps {
|
|||||||
tabs: {
|
tabs: {
|
||||||
remove: string;
|
remove: string;
|
||||||
add: string;
|
add: string;
|
||||||
|
removeRepo: string;
|
||||||
|
};
|
||||||
|
removeRepoForm: {
|
||||||
|
repoUrl: string;
|
||||||
|
repoUrlPlaceholder: string;
|
||||||
|
repoUrlHelp: string;
|
||||||
|
reason: string;
|
||||||
|
reasonPlaceholder: string;
|
||||||
|
submit: string;
|
||||||
|
submitting: string;
|
||||||
|
};
|
||||||
|
removeRepoSuccess: {
|
||||||
|
title: string;
|
||||||
|
description: string;
|
||||||
|
descriptionZero: string;
|
||||||
|
};
|
||||||
|
removeRepoError: {
|
||||||
|
notOwner: string;
|
||||||
|
invalidRepo: string;
|
||||||
|
parseError: string;
|
||||||
};
|
};
|
||||||
form: {
|
form: {
|
||||||
skillId: string;
|
skillId: string;
|
||||||
@@ -69,6 +89,8 @@ interface ClaimFormProps {
|
|||||||
foundSkillsIn: string;
|
foundSkillsIn: string;
|
||||||
root: string;
|
root: string;
|
||||||
andMore: string;
|
andMore: string;
|
||||||
|
reEnabledTitle: string;
|
||||||
|
reEnabledDescription: string;
|
||||||
};
|
};
|
||||||
error: {
|
error: {
|
||||||
notOwner: string;
|
notOwner: string;
|
||||||
@@ -81,6 +103,7 @@ interface ClaimFormProps {
|
|||||||
rateLimitExceeded: string;
|
rateLimitExceeded: string;
|
||||||
networkTimeout: string;
|
networkTimeout: string;
|
||||||
generic: string;
|
generic: string;
|
||||||
|
repoBlockedByOwner: string;
|
||||||
};
|
};
|
||||||
myRequests: {
|
myRequests: {
|
||||||
title: string;
|
title: string;
|
||||||
@@ -122,15 +145,15 @@ interface AddRequest {
|
|||||||
|
|
||||||
export function ClaimForm({ translations }: ClaimFormProps) {
|
export function ClaimForm({ translations }: ClaimFormProps) {
|
||||||
const { data: session, status } = useSession();
|
const { data: session, status } = useSession();
|
||||||
const [activeTab, setActiveTab] = useState<'remove' | 'add'>('add'); // Default to 'add' tab
|
const [activeTab, setActiveTab] = useState<'remove' | 'add' | 'remove-repo'>('add'); // Default to 'add' tab
|
||||||
const [expandedRequests, setExpandedRequests] = useState<Set<string>>(new Set());
|
const [expandedRequests, setExpandedRequests] = useState<Set<string>>(new Set());
|
||||||
|
|
||||||
// Read tab from URL hash on mount and update on hash change
|
// Read tab from URL hash on mount and update on hash change
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const readHashTab = () => {
|
const readHashTab = () => {
|
||||||
const hash = window.location.hash.slice(1);
|
const hash = window.location.hash.slice(1);
|
||||||
if (hash === 'remove' || hash === 'add') {
|
if (hash === 'remove' || hash === 'add' || hash === 'remove-repo') {
|
||||||
setActiveTab(hash);
|
setActiveTab(hash as 'remove' | 'add' | 'remove-repo');
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
readHashTab();
|
readHashTab();
|
||||||
@@ -139,7 +162,7 @@ export function ClaimForm({ translations }: ClaimFormProps) {
|
|||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
// Update URL hash when tab changes
|
// Update URL hash when tab changes
|
||||||
const handleTabChange = useCallback((tab: 'remove' | 'add') => {
|
const handleTabChange = useCallback((tab: 'remove' | 'add' | 'remove-repo') => {
|
||||||
setActiveTab(tab);
|
setActiveTab(tab);
|
||||||
window.history.replaceState(null, '', `#${tab}`);
|
window.history.replaceState(null, '', `#${tab}`);
|
||||||
}, []);
|
}, []);
|
||||||
@@ -177,9 +200,19 @@ export function ClaimForm({ translations }: ClaimFormProps) {
|
|||||||
const [addHasSkillMd, setAddHasSkillMd] = useState(true);
|
const [addHasSkillMd, setAddHasSkillMd] = useState(true);
|
||||||
const [addSkillCount, setAddSkillCount] = useState(0);
|
const [addSkillCount, setAddSkillCount] = useState(0);
|
||||||
const [addSkillPaths, setAddSkillPaths] = useState<string[]>([]);
|
const [addSkillPaths, setAddSkillPaths] = useState<string[]>([]);
|
||||||
|
const [addReEnabled, setAddReEnabled] = useState(false);
|
||||||
const [addRequests, setAddRequests] = useState<AddRequest[]>([]);
|
const [addRequests, setAddRequests] = useState<AddRequest[]>([]);
|
||||||
const [loadingAddRequests, setLoadingAddRequests] = useState(false);
|
const [loadingAddRequests, setLoadingAddRequests] = useState(false);
|
||||||
|
|
||||||
|
// Remove-repo form state
|
||||||
|
const [repoUrl, setRepoUrl] = useState('');
|
||||||
|
const [repoReason, setRepoReason] = useState('');
|
||||||
|
const [isSubmittingRepo, setIsSubmittingRepo] = useState(false);
|
||||||
|
const [repoError, setRepoError] = useState('');
|
||||||
|
const [repoSuccess, setRepoSuccess] = useState(false);
|
||||||
|
const [repoBlockedCount, setRepoBlockedCount] = useState(0);
|
||||||
|
const [repoName, setRepoName] = useState('');
|
||||||
|
|
||||||
// Fetch user's existing requests
|
// Fetch user's existing requests
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (session) {
|
if (session) {
|
||||||
@@ -331,6 +364,9 @@ export function ClaimForm({ translations }: ClaimFormProps) {
|
|||||||
case 'ALREADY_PENDING':
|
case 'ALREADY_PENDING':
|
||||||
setAddError(translations.error.alreadyPending);
|
setAddError(translations.error.alreadyPending);
|
||||||
break;
|
break;
|
||||||
|
case 'REPO_BLOCKED_BY_OWNER':
|
||||||
|
setAddError(translations.error.repoBlockedByOwner);
|
||||||
|
break;
|
||||||
case 'AUTH_REQUIRED':
|
case 'AUTH_REQUIRED':
|
||||||
signIn('github');
|
signIn('github');
|
||||||
return;
|
return;
|
||||||
@@ -342,6 +378,7 @@ export function ClaimForm({ translations }: ClaimFormProps) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
setAddSuccess(true);
|
setAddSuccess(true);
|
||||||
|
setAddReEnabled(data.reEnabled ?? false);
|
||||||
setAddHasSkillMd(data.hasSkillMd ?? true);
|
setAddHasSkillMd(data.hasSkillMd ?? true);
|
||||||
setAddSkillCount(data.skillCount ?? 0);
|
setAddSkillCount(data.skillCount ?? 0);
|
||||||
setAddSkillPaths(data.skillPaths ?? []);
|
setAddSkillPaths(data.skillPaths ?? []);
|
||||||
@@ -355,6 +392,61 @@ export function ClaimForm({ translations }: ClaimFormProps) {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const handleRepoRemovalSubmit = async (e: React.FormEvent) => {
|
||||||
|
e.preventDefault();
|
||||||
|
if (!session) {
|
||||||
|
signIn('github');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!repoUrl.trim()) return;
|
||||||
|
|
||||||
|
setIsSubmittingRepo(true);
|
||||||
|
setRepoError('');
|
||||||
|
setRepoSuccess(false);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const res = await fetchWithCsrf('/api/skills/repo-removal-request', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ repoUrl: repoUrl.trim(), reason: repoReason.trim() }),
|
||||||
|
});
|
||||||
|
|
||||||
|
const data = await res.json();
|
||||||
|
|
||||||
|
if (!res.ok) {
|
||||||
|
switch (data.code) {
|
||||||
|
case 'NOT_OWNER':
|
||||||
|
setRepoError(translations.removeRepoError.notOwner);
|
||||||
|
break;
|
||||||
|
case 'INVALID_REPO':
|
||||||
|
setRepoError(translations.removeRepoError.invalidRepo);
|
||||||
|
break;
|
||||||
|
case 'PARSE_ERROR':
|
||||||
|
setRepoError(translations.removeRepoError.parseError);
|
||||||
|
break;
|
||||||
|
case 'AUTH_REQUIRED':
|
||||||
|
signIn('github');
|
||||||
|
return;
|
||||||
|
default:
|
||||||
|
console.error('Repo removal error:', data);
|
||||||
|
setRepoError(translations.error.generic);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
setRepoSuccess(true);
|
||||||
|
setRepoBlockedCount(data.blockedCount ?? 0);
|
||||||
|
setRepoName(data.repo ?? repoUrl.trim());
|
||||||
|
setRepoUrl('');
|
||||||
|
setRepoReason('');
|
||||||
|
} catch {
|
||||||
|
setRepoError(translations.error.generic);
|
||||||
|
} finally {
|
||||||
|
setIsSubmittingRepo(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const getStatusBadge = (requestStatus: string) => {
|
const getStatusBadge = (requestStatus: string) => {
|
||||||
switch (requestStatus) {
|
switch (requestStatus) {
|
||||||
case 'pending':
|
case 'pending':
|
||||||
@@ -457,11 +549,40 @@ export function ClaimForm({ translations }: ClaimFormProps) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Repo removal success state
|
||||||
|
if (repoSuccess) {
|
||||||
|
return (
|
||||||
|
<div className="card p-8 text-center">
|
||||||
|
<CheckCircle className="w-12 h-12 mx-auto mb-4 text-success" />
|
||||||
|
<h2 className="text-xl font-semibold text-text-primary mb-2">
|
||||||
|
{translations.removeRepoSuccess.title}
|
||||||
|
</h2>
|
||||||
|
<p className="text-text-secondary mb-6">
|
||||||
|
{repoBlockedCount > 0
|
||||||
|
? translations.removeRepoSuccess.description
|
||||||
|
.replace('{count}', String(repoBlockedCount))
|
||||||
|
.replace('{repo}', repoName)
|
||||||
|
: translations.removeRepoSuccess.descriptionZero}
|
||||||
|
</p>
|
||||||
|
<button
|
||||||
|
onClick={() => { setRepoSuccess(false); setRepoBlockedCount(0); setRepoName(''); }}
|
||||||
|
className="btn-secondary"
|
||||||
|
>
|
||||||
|
{translations.success.viewRequests}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// Add success state
|
// Add success state
|
||||||
if (addSuccess) {
|
if (addSuccess) {
|
||||||
// Determine which message to show
|
// Determine which message to show
|
||||||
|
let successTitle = translations.addSuccess.title;
|
||||||
let successDescription: string | React.ReactNode;
|
let successDescription: string | React.ReactNode;
|
||||||
if (addSkillCount > 1) {
|
if (addReEnabled) {
|
||||||
|
successTitle = translations.addSuccess.reEnabledTitle;
|
||||||
|
successDescription = translations.addSuccess.reEnabledDescription;
|
||||||
|
} else if (addSkillCount > 1) {
|
||||||
successDescription = (
|
successDescription = (
|
||||||
<>
|
<>
|
||||||
{translations.addSuccess.descriptionMultiplePrefix}
|
{translations.addSuccess.descriptionMultiplePrefix}
|
||||||
@@ -479,7 +600,7 @@ export function ClaimForm({ translations }: ClaimFormProps) {
|
|||||||
<div className="card p-8 text-center">
|
<div className="card p-8 text-center">
|
||||||
<CheckCircle className="w-12 h-12 mx-auto mb-4 text-success" />
|
<CheckCircle className="w-12 h-12 mx-auto mb-4 text-success" />
|
||||||
<h2 className="text-xl font-semibold text-text-primary mb-2">
|
<h2 className="text-xl font-semibold text-text-primary mb-2">
|
||||||
{translations.addSuccess.title}
|
{successTitle}
|
||||||
</h2>
|
</h2>
|
||||||
<p className="text-text-secondary mb-4">
|
<p className="text-text-secondary mb-4">
|
||||||
{successDescription}
|
{successDescription}
|
||||||
@@ -500,7 +621,7 @@ export function ClaimForm({ translations }: ClaimFormProps) {
|
|||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
<button
|
<button
|
||||||
onClick={() => setAddSuccess(false)}
|
onClick={() => { setAddSuccess(false); setAddReEnabled(false); }}
|
||||||
className="btn-secondary"
|
className="btn-secondary"
|
||||||
>
|
>
|
||||||
{translations.addSuccess.viewRequests}
|
{translations.addSuccess.viewRequests}
|
||||||
@@ -535,6 +656,17 @@ export function ClaimForm({ translations }: ClaimFormProps) {
|
|||||||
<Plus className="w-4 h-4" />
|
<Plus className="w-4 h-4" />
|
||||||
{translations.tabs.add}
|
{translations.tabs.add}
|
||||||
</button>
|
</button>
|
||||||
|
<button
|
||||||
|
onClick={() => handleTabChange('remove-repo')}
|
||||||
|
className={`flex items-center gap-2 px-4 py-3 text-sm font-medium border-b-2 transition-colors ${
|
||||||
|
activeTab === 'remove-repo'
|
||||||
|
? 'border-error text-error'
|
||||||
|
: 'border-transparent text-text-muted hover:text-text-primary'
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
<Trash2 className="w-4 h-4" />
|
||||||
|
{translations.tabs.removeRepo}
|
||||||
|
</button>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Remove Form */}
|
{/* Remove Form */}
|
||||||
@@ -652,6 +784,79 @@ export function ClaimForm({ translations }: ClaimFormProps) {
|
|||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
{/* Remove Repository Form */}
|
||||||
|
{activeTab === 'remove-repo' && (
|
||||||
|
<div className="card p-6">
|
||||||
|
<form onSubmit={handleRepoRemovalSubmit} className="space-y-4">
|
||||||
|
<div>
|
||||||
|
<label
|
||||||
|
htmlFor="repoUrl"
|
||||||
|
className="block text-sm font-medium text-text-primary mb-2"
|
||||||
|
>
|
||||||
|
{translations.removeRepoForm.repoUrl}
|
||||||
|
</label>
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
id="repoUrl"
|
||||||
|
value={repoUrl}
|
||||||
|
onChange={(e) => setRepoUrl(e.target.value)}
|
||||||
|
placeholder={translations.removeRepoForm.repoUrlPlaceholder}
|
||||||
|
className="w-full px-4 py-2 border border-border rounded-lg bg-surface text-text-primary placeholder-text-muted focus:outline-none focus:ring-2 focus:ring-error"
|
||||||
|
dir="ltr"
|
||||||
|
required
|
||||||
|
/>
|
||||||
|
<p className="text-xs text-text-muted mt-1">
|
||||||
|
{translations.removeRepoForm.repoUrlHelp}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<label
|
||||||
|
htmlFor="repoReason"
|
||||||
|
className="block text-sm font-medium text-text-primary mb-2"
|
||||||
|
>
|
||||||
|
{translations.removeRepoForm.reason} <span className="text-text-muted">{translations.optional}</span>
|
||||||
|
</label>
|
||||||
|
<textarea
|
||||||
|
id="repoReason"
|
||||||
|
value={repoReason}
|
||||||
|
onChange={(e) => setRepoReason(e.target.value)}
|
||||||
|
placeholder={translations.removeRepoForm.reasonPlaceholder}
|
||||||
|
rows={3}
|
||||||
|
className="w-full px-4 py-2 border border-border rounded-lg bg-surface text-text-primary placeholder-text-muted focus:outline-none focus:ring-2 focus:ring-primary-500 resize-none"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{repoError && (
|
||||||
|
<div className="p-3 bg-error-bg border border-error/30 rounded-lg">
|
||||||
|
<div className="flex items-center gap-2 text-error">
|
||||||
|
<AlertCircle className="w-4 h-4" />
|
||||||
|
<p className="text-sm">{repoError}</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<button
|
||||||
|
type="submit"
|
||||||
|
disabled={isSubmittingRepo || !repoUrl.trim()}
|
||||||
|
className="w-full justify-center inline-flex items-center gap-2 px-4 py-2 bg-error text-white rounded-lg font-medium hover:bg-error/90 transition-colors disabled:opacity-50"
|
||||||
|
>
|
||||||
|
{isSubmittingRepo ? (
|
||||||
|
<>
|
||||||
|
<Loader2 className="w-4 h-4 animate-spin" />
|
||||||
|
{translations.removeRepoForm.submitting}
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
<Trash2 className="w-4 h-4" />
|
||||||
|
{translations.removeRepoForm.submit}
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</button>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
{/* Add Form */}
|
{/* Add Form */}
|
||||||
{activeTab === 'add' && (
|
{activeTab === 'add' && (
|
||||||
<>
|
<>
|
||||||
|
|||||||
@@ -545,6 +545,8 @@
|
|||||||
"browseAll": "Browse all skills",
|
"browseAll": "Browse all skills",
|
||||||
"claimCta": "Is this your repository?",
|
"claimCta": "Is this your repository?",
|
||||||
"claimButton": "Claim ownership",
|
"claimButton": "Claim ownership",
|
||||||
|
"ownerManageCta": "You own these skills.",
|
||||||
|
"ownerManageButton": "Manage / Remove Skills",
|
||||||
"installCta": "Install the most popular skill from this owner:",
|
"installCta": "Install the most popular skill from this owner:",
|
||||||
"stats": {
|
"stats": {
|
||||||
"skills": "{count, plural, one {# skill} other {# skills}}",
|
"skills": "{count, plural, one {# skill} other {# skills}}",
|
||||||
@@ -696,7 +698,8 @@
|
|||||||
},
|
},
|
||||||
"tabs": {
|
"tabs": {
|
||||||
"remove": "Remove Skill",
|
"remove": "Remove Skill",
|
||||||
"add": "Add Skill"
|
"add": "Add Skill",
|
||||||
|
"removeRepo": "Remove Repository"
|
||||||
},
|
},
|
||||||
"form": {
|
"form": {
|
||||||
"skillId": "Skill ID",
|
"skillId": "Skill ID",
|
||||||
@@ -732,7 +735,9 @@
|
|||||||
"viewRequests": "View My Requests",
|
"viewRequests": "View My Requests",
|
||||||
"foundSkillsIn": "Found skills in:",
|
"foundSkillsIn": "Found skills in:",
|
||||||
"root": "(root)",
|
"root": "(root)",
|
||||||
"andMore": "... and {count} more"
|
"andMore": "... and {count} more",
|
||||||
|
"reEnabledTitle": "Repository Re-enabled",
|
||||||
|
"reEnabledDescription": "Your repository has been unblocked and will appear again after the next scan."
|
||||||
},
|
},
|
||||||
"error": {
|
"error": {
|
||||||
"notOwner": "You are not the owner or admin of this repository. Only repository owners can request removal.",
|
"notOwner": "You are not the owner or admin of this repository. Only repository owners can request removal.",
|
||||||
@@ -744,7 +749,8 @@
|
|||||||
"invalidRepo": "The repository was not found or is not accessible. Please check the URL and ensure the repository is public.",
|
"invalidRepo": "The repository was not found or is not accessible. Please check the URL and ensure the repository is public.",
|
||||||
"rateLimitExceeded": "GitHub API rate limit exceeded. Please try again in a few minutes.",
|
"rateLimitExceeded": "GitHub API rate limit exceeded. Please try again in a few minutes.",
|
||||||
"networkTimeout": "Request timed out while checking the repository. Please try again.",
|
"networkTimeout": "Request timed out while checking the repository. Please try again.",
|
||||||
"generic": "An error occurred. Please try again."
|
"generic": "An error occurred. Please try again.",
|
||||||
|
"repoBlockedByOwner": "This repository was removed by its owner and cannot be re-added."
|
||||||
},
|
},
|
||||||
"myRequests": {
|
"myRequests": {
|
||||||
"title": "My Requests",
|
"title": "My Requests",
|
||||||
@@ -760,6 +766,25 @@
|
|||||||
"showLess": "Show less",
|
"showLess": "Show less",
|
||||||
"showAllPrefix": "Show all ",
|
"showAllPrefix": "Show all ",
|
||||||
"showAllSuffix": " paths"
|
"showAllSuffix": " paths"
|
||||||
|
},
|
||||||
|
"removeRepoForm": {
|
||||||
|
"repoUrl": "GitHub Repository URL or owner/repo",
|
||||||
|
"repoUrlPlaceholder": "https://github.com/owner/repo",
|
||||||
|
"repoUrlHelp": "All skills from this repository will be removed from SkillHub and it will not be re-indexed.",
|
||||||
|
"reason": "Reason for removal",
|
||||||
|
"reasonPlaceholder": "Why are you removing this repository?",
|
||||||
|
"submit": "Remove All Skills",
|
||||||
|
"submitting": "Removing..."
|
||||||
|
},
|
||||||
|
"removeRepoSuccess": {
|
||||||
|
"title": "Repository Removed",
|
||||||
|
"description": "All {count} skills from {repo} have been removed from SkillHub.",
|
||||||
|
"descriptionZero": "No active skills were found for this repository. It has been blocked from re-indexing."
|
||||||
|
},
|
||||||
|
"removeRepoError": {
|
||||||
|
"notOwner": "You are not the owner of this repository.",
|
||||||
|
"invalidRepo": "Repository not found on GitHub.",
|
||||||
|
"parseError": "Invalid format. Use https://github.com/owner/repo or owner/repo."
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"attribution": {
|
"attribution": {
|
||||||
|
|||||||
@@ -545,6 +545,8 @@
|
|||||||
"browseAll": "مرور همه مهارتها",
|
"browseAll": "مرور همه مهارتها",
|
||||||
"claimCta": "آیا این مخزن شماست؟",
|
"claimCta": "آیا این مخزن شماست؟",
|
||||||
"claimButton": "ثبت مالکیت",
|
"claimButton": "ثبت مالکیت",
|
||||||
|
"ownerManageCta": "این مهارتها متعلق به شما هستند.",
|
||||||
|
"ownerManageButton": "مدیریت / حذف مهارتها",
|
||||||
"installCta": "نصب محبوبترین مهارت از این مالک:",
|
"installCta": "نصب محبوبترین مهارت از این مالک:",
|
||||||
"stats": {
|
"stats": {
|
||||||
"skills": "{count, plural, one {# مهارت} other {# مهارت}}",
|
"skills": "{count, plural, one {# مهارت} other {# مهارت}}",
|
||||||
@@ -696,7 +698,8 @@
|
|||||||
},
|
},
|
||||||
"tabs": {
|
"tabs": {
|
||||||
"remove": "حذف مهارت",
|
"remove": "حذف مهارت",
|
||||||
"add": "افزودن مهارت"
|
"add": "افزودن مهارت",
|
||||||
|
"removeRepo": "حذف مخزن"
|
||||||
},
|
},
|
||||||
"form": {
|
"form": {
|
||||||
"skillId": "شناسه مهارت",
|
"skillId": "شناسه مهارت",
|
||||||
@@ -732,7 +735,9 @@
|
|||||||
"viewRequests": "مشاهده درخواستها",
|
"viewRequests": "مشاهده درخواستها",
|
||||||
"foundSkillsIn": "مهارتها در این مسیرها یافت شدند:",
|
"foundSkillsIn": "مهارتها در این مسیرها یافت شدند:",
|
||||||
"root": "(root)",
|
"root": "(root)",
|
||||||
"andMore": "... و {count} مورد دیگر"
|
"andMore": "... و {count} مورد دیگر",
|
||||||
|
"reEnabledTitle": "مخزن فعالسازی مجدد شد",
|
||||||
|
"reEnabledDescription": "مخزن شما رفع انسداد شد و پس از اسکن بعدی دوباره نمایش داده خواهد شد."
|
||||||
},
|
},
|
||||||
"error": {
|
"error": {
|
||||||
"notOwner": "شما مالک یا ادمین این مخزن نیستید. فقط مالکان مخزن میتوانند درخواست حذف دهند.",
|
"notOwner": "شما مالک یا ادمین این مخزن نیستید. فقط مالکان مخزن میتوانند درخواست حذف دهند.",
|
||||||
@@ -744,7 +749,8 @@
|
|||||||
"invalidRepo": "مخزن یافت نشد یا قابل دسترسی نیست. لطفاً URL را بررسی کنید و اطمینان حاصل کنید که مخزن عمومی است.",
|
"invalidRepo": "مخزن یافت نشد یا قابل دسترسی نیست. لطفاً URL را بررسی کنید و اطمینان حاصل کنید که مخزن عمومی است.",
|
||||||
"rateLimitExceeded": "محدودیت API GitHub تمام شده است. لطفاً چند دقیقه بعد دوباره تلاش کنید.",
|
"rateLimitExceeded": "محدودیت API GitHub تمام شده است. لطفاً چند دقیقه بعد دوباره تلاش کنید.",
|
||||||
"networkTimeout": "هنگام بررسی مخزن زمان اتصال تمام شد. لطفاً دوباره تلاش کنید.",
|
"networkTimeout": "هنگام بررسی مخزن زمان اتصال تمام شد. لطفاً دوباره تلاش کنید.",
|
||||||
"generic": "خطایی رخ داد. لطفاً دوباره تلاش کنید."
|
"generic": "خطایی رخ داد. لطفاً دوباره تلاش کنید.",
|
||||||
|
"repoBlockedByOwner": "این مخزن توسط مالکش حذف شده و قابل افزودن مجدد نیست."
|
||||||
},
|
},
|
||||||
"myRequests": {
|
"myRequests": {
|
||||||
"title": "درخواستهای من",
|
"title": "درخواستهای من",
|
||||||
@@ -760,6 +766,25 @@
|
|||||||
"showLess": "نمایش کمتر",
|
"showLess": "نمایش کمتر",
|
||||||
"showAllPrefix": "نمایش همه ",
|
"showAllPrefix": "نمایش همه ",
|
||||||
"showAllSuffix": " مسیر"
|
"showAllSuffix": " مسیر"
|
||||||
|
},
|
||||||
|
"removeRepoForm": {
|
||||||
|
"repoUrl": "آدرس مخزن گیتهاب یا owner/repo",
|
||||||
|
"repoUrlPlaceholder": "https://github.com/owner/repo",
|
||||||
|
"repoUrlHelp": "تمام مهارتهای این مخزن از SkillHub حذف شده و دیگر ایندکس نخواهد شد.",
|
||||||
|
"reason": "دلیل حذف",
|
||||||
|
"reasonPlaceholder": "چرا این مخزن را حذف میکنید؟",
|
||||||
|
"submit": "حذف تمام مهارتها",
|
||||||
|
"submitting": "در حال حذف..."
|
||||||
|
},
|
||||||
|
"removeRepoSuccess": {
|
||||||
|
"title": "مخزن حذف شد",
|
||||||
|
"description": "تمام {count} مهارت از {repo} از SkillHub حذف شدند.",
|
||||||
|
"descriptionZero": "هیچ مهارت فعالی برای این مخزن یافت نشد. مخزن از ایندکسسازی مجدد مسدود شده است."
|
||||||
|
},
|
||||||
|
"removeRepoError": {
|
||||||
|
"notOwner": "شما مالک این مخزن نیستید.",
|
||||||
|
"invalidRepo": "مخزن در گیتهاب یافت نشد.",
|
||||||
|
"parseError": "فرمت نادرست. از https://github.com/owner/repo یا owner/repo استفاده کنید."
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"attribution": {
|
"attribution": {
|
||||||
|
|||||||
@@ -83,6 +83,9 @@ export const config = {
|
|||||||
'/(en|fa)/:path*',
|
'/(en|fa)/:path*',
|
||||||
// Match API routes (for CSRF protection)
|
// Match API routes (for CSRF protection)
|
||||||
'/api/:path*',
|
'/api/:path*',
|
||||||
|
// Match skill paths explicitly (repo names like "next.js" contain dots
|
||||||
|
// which would be excluded by the catch-all pattern below)
|
||||||
|
'/skill/:path*',
|
||||||
// Match other paths (exclude static files and Next.js internals)
|
// Match other paths (exclude static files and Next.js internals)
|
||||||
'/((?!_next|_vercel|.*\\..*).*)',
|
'/((?!_next|_vercel|.*\\..*).*)',
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -43,6 +43,20 @@ Sentry.init({
|
|||||||
/Object Not Found Matching Id/,
|
/Object Not Found Matching Id/,
|
||||||
// ResizeObserver loop limit — browser quirk, not actionable
|
// ResizeObserver loop limit — browser quirk, not actionable
|
||||||
"ResizeObserver",
|
"ResizeObserver",
|
||||||
|
// DOM manipulation by browser extensions (translate, font, etc.) conflicts with React
|
||||||
|
"NotFoundError: Failed to execute 'removeChild' on 'Node'",
|
||||||
|
"NotFoundError: Failed to execute 'insertBefore' on 'Node'",
|
||||||
|
// Browser extension internal errors (crypto wallets, etc.)
|
||||||
|
/func .* not found/,
|
||||||
|
],
|
||||||
|
|
||||||
|
// Ignore errors originating from browser extensions or injected scripts
|
||||||
|
denyUrls: [
|
||||||
|
/extensions\//i,
|
||||||
|
/^chrome:\/\//i,
|
||||||
|
/^chrome-extension:\/\//i,
|
||||||
|
/^moz-extension:\/\//i,
|
||||||
|
/inpage\.js/,
|
||||||
],
|
],
|
||||||
|
|
||||||
beforeSend(event, hint) {
|
beforeSend(event, hint) {
|
||||||
|
|||||||
@@ -121,8 +121,14 @@ services:
|
|||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
postgres_data:
|
postgres_data:
|
||||||
|
name: skillhub-public_postgres_data
|
||||||
|
external: true
|
||||||
redis_data:
|
redis_data:
|
||||||
|
name: skillhub-public_redis_data
|
||||||
|
external: true
|
||||||
meilisearch_data:
|
meilisearch_data:
|
||||||
|
name: skillhub-public_meilisearch_data
|
||||||
|
external: true
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
default:
|
default:
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import {
|
|||||||
ratingQueries,
|
ratingQueries,
|
||||||
installationQueries,
|
installationQueries,
|
||||||
favoriteQueries,
|
favoriteQueries,
|
||||||
|
discoveredRepoQueries,
|
||||||
} from './queries.js';
|
} from './queries.js';
|
||||||
import {
|
import {
|
||||||
createTestSkill,
|
createTestSkill,
|
||||||
@@ -22,13 +23,14 @@ import {
|
|||||||
|
|
||||||
// Mock the schema imports
|
// Mock the schema imports
|
||||||
vi.mock('./schema.js', () => ({
|
vi.mock('./schema.js', () => ({
|
||||||
skills: { id: 'id', name: 'name', description: 'description', githubStars: 'github_stars', downloadCount: 'download_count', rating: 'rating', updatedAt: 'updated_at', isFeatured: 'is_featured', isVerified: 'is_verified', securityScore: 'security_score', viewCount: 'view_count', ratingCount: 'rating_count', ratingSum: 'rating_sum' },
|
skills: { id: 'id', name: 'name', description: 'description', githubStars: 'github_stars', downloadCount: 'download_count', rating: 'rating', updatedAt: 'updated_at', isFeatured: 'is_featured', isVerified: 'is_verified', securityScore: 'security_score', viewCount: 'view_count', ratingCount: 'rating_count', ratingSum: 'rating_sum', githubOwner: 'github_owner', githubRepo: 'github_repo', isBlocked: 'is_blocked', isOwnerClaimed: 'is_owner_claimed', isDuplicate: 'is_duplicate' },
|
||||||
categories: { id: 'id', name: 'name', slug: 'slug', sortOrder: 'sort_order', skillCount: 'skill_count' },
|
categories: { id: 'id', name: 'name', slug: 'slug', sortOrder: 'sort_order', skillCount: 'skill_count' },
|
||||||
skillCategories: { skillId: 'skill_id', categoryId: 'category_id' },
|
skillCategories: { skillId: 'skill_id', categoryId: 'category_id' },
|
||||||
users: { id: 'id', githubId: 'github_id', username: 'username', avatarUrl: 'avatar_url' },
|
users: { id: 'id', githubId: 'github_id', username: 'username', avatarUrl: 'avatar_url' },
|
||||||
ratings: { id: 'id', skillId: 'skill_id', userId: 'user_id', rating: 'rating', createdAt: 'created_at' },
|
ratings: { id: 'id', skillId: 'skill_id', userId: 'user_id', rating: 'rating', createdAt: 'created_at' },
|
||||||
installations: { id: 'id', skillId: 'skill_id', platform: 'platform', method: 'method' },
|
installations: { id: 'id', skillId: 'skill_id', platform: 'platform', method: 'method' },
|
||||||
favorites: { userId: 'user_id', skillId: 'skill_id', createdAt: 'created_at' },
|
favorites: { userId: 'user_id', skillId: 'skill_id', createdAt: 'created_at' },
|
||||||
|
discoveredRepos: { id: 'id', isBlocked: 'is_blocked', isArchived: 'is_archived', lastScanned: 'last_scanned', githubStars: 'github_stars', discoveredVia: 'discovered_via', owner: 'owner', repo: 'repo', githubForks: 'github_forks', defaultBranch: 'default_branch', hasSkillMd: 'has_skill_md', skillCount: 'skill_count', scanError: 'scan_error', updatedAt: 'updated_at' },
|
||||||
}));
|
}));
|
||||||
|
|
||||||
// Helper to create a chainable mock
|
// Helper to create a chainable mock
|
||||||
@@ -659,3 +661,127 @@ describe('favoriteQueries', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('skillQueries (repo block/unblock)', () => {
|
||||||
|
describe('blockByRepo', () => {
|
||||||
|
it('should call update with isBlocked: true for matching owner/repo', async () => {
|
||||||
|
const mockDb = createMockDb();
|
||||||
|
|
||||||
|
await skillQueries.blockByRepo(mockDb as any, 'rawveg', 'skillsforge-marketplace');
|
||||||
|
|
||||||
|
expect(mockDb.update).toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('unblockByRepo', () => {
|
||||||
|
it('should call update with isBlocked: false for matching owner/repo', async () => {
|
||||||
|
const mockDb = createMockDb();
|
||||||
|
|
||||||
|
await skillQueries.unblockByRepo(mockDb as any, 'rawveg', 'skillsforge-marketplace');
|
||||||
|
|
||||||
|
expect(mockDb.update).toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('setOwnerClaimed', () => {
|
||||||
|
it('should call update with isOwnerClaimed: true for matching owner/repo', async () => {
|
||||||
|
const mockDb = createMockDb();
|
||||||
|
|
||||||
|
await skillQueries.setOwnerClaimed(mockDb as any, 'rawveg', 'skillsforge-marketplace');
|
||||||
|
|
||||||
|
expect(mockDb.update).toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('countByRepo', () => {
|
||||||
|
it('should return count of non-blocked skills for owner/repo', async () => {
|
||||||
|
const mockDb = createMockDb([{ count: 16 }]);
|
||||||
|
|
||||||
|
const result = await skillQueries.countByRepo(mockDb as any, 'rawveg', 'skillsforge-marketplace');
|
||||||
|
|
||||||
|
expect(result).toBe(16);
|
||||||
|
expect(mockDb.select).toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return 0 when no matching skills', async () => {
|
||||||
|
const mockDb = createMockDb([{ count: 0 }]);
|
||||||
|
|
||||||
|
const result = await skillQueries.countByRepo(mockDb as any, 'unknown', 'unknown-repo');
|
||||||
|
|
||||||
|
expect(result).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return 0 when result is empty', async () => {
|
||||||
|
const mockDb = createMockDb([]);
|
||||||
|
|
||||||
|
const result = await skillQueries.countByRepo(mockDb as any, 'owner', 'repo');
|
||||||
|
|
||||||
|
expect(result).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('discoveredRepoQueries', () => {
|
||||||
|
describe('blockRepo', () => {
|
||||||
|
it('should call update with isBlocked: true for the repo id', async () => {
|
||||||
|
const mockDb = createMockDb();
|
||||||
|
|
||||||
|
await discoveredRepoQueries.blockRepo(mockDb as any, 'rawveg/skillsforge-marketplace');
|
||||||
|
|
||||||
|
expect(mockDb.update).toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('unblockRepo', () => {
|
||||||
|
it('should call update with isBlocked: false for the repo id', async () => {
|
||||||
|
const mockDb = createMockDb();
|
||||||
|
|
||||||
|
await discoveredRepoQueries.unblockRepo(mockDb as any, 'rawveg/skillsforge-marketplace');
|
||||||
|
|
||||||
|
expect(mockDb.update).toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('getNeedingScanning', () => {
|
||||||
|
it('should return repos that need scanning', async () => {
|
||||||
|
const repos = [
|
||||||
|
{ id: 'owner/repo1', isBlocked: false },
|
||||||
|
{ id: 'owner/repo2', isBlocked: false },
|
||||||
|
];
|
||||||
|
const mockDb = createMockDb(repos);
|
||||||
|
|
||||||
|
const result = await discoveredRepoQueries.getNeedingScanning(mockDb as any);
|
||||||
|
|
||||||
|
expect(result).toEqual(repos);
|
||||||
|
expect(mockDb.select).toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return empty array when no repos need scanning', async () => {
|
||||||
|
const mockDb = createMockDb([]);
|
||||||
|
|
||||||
|
const result = await discoveredRepoQueries.getNeedingScanning(mockDb as any);
|
||||||
|
|
||||||
|
expect(result).toEqual([]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('getById', () => {
|
||||||
|
it('should return repo when found', async () => {
|
||||||
|
const repo = { id: 'rawveg/skillsforge-marketplace', isBlocked: true };
|
||||||
|
const mockDb = createMockDb([repo]);
|
||||||
|
|
||||||
|
const result = await discoveredRepoQueries.getById(mockDb as any, 'rawveg/skillsforge-marketplace');
|
||||||
|
|
||||||
|
expect(result).toEqual(repo);
|
||||||
|
expect(mockDb.select).toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return null when repo not found', async () => {
|
||||||
|
const mockDb = createMockDb([]);
|
||||||
|
|
||||||
|
const result = await discoveredRepoQueries.getById(mockDb as any, 'nonexistent/repo');
|
||||||
|
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ type DB = PostgresJsDatabase<typeof schema>;
|
|||||||
* Skills with skill_type=NULL (newly crawled, not yet curated) are included
|
* Skills with skill_type=NULL (newly crawled, not yet curated) are included
|
||||||
* so they don't disappear between curate.mjs runs.
|
* so they don't disappear between curate.mjs runs.
|
||||||
*/
|
*/
|
||||||
const browseReadyFilter = sql`(${skills.isDuplicate} = false)`;
|
const browseReadyFilter = sql`(${skills.isDuplicate} = false OR ${skills.isOwnerClaimed} = true) AND ${skills.isStale} = false`;
|
||||||
|
|
||||||
/** Minimum repo age gap (in days) to trust repo_created_at over stars for duplicate detection */
|
/** Minimum repo age gap (in days) to trust repo_created_at over stars for duplicate detection */
|
||||||
const REPO_AGE_THRESHOLD_DAYS = 75;
|
const REPO_AGE_THRESHOLD_DAYS = 75;
|
||||||
@@ -688,6 +688,10 @@ export const skillQueries = {
|
|||||||
: (skill.securityStatus && skill.qualityScore != null)
|
: (skill.securityStatus && skill.qualityScore != null)
|
||||||
? { reviewStatus: sql`CASE WHEN ${skills.reviewStatus} IS NULL OR ${skills.reviewStatus} = 'unreviewed' THEN 'auto-scored' ELSE ${skills.reviewStatus} END` }
|
? { reviewStatus: sql`CASE WHEN ${skills.reviewStatus} IS NULL OR ${skills.reviewStatus} = 'unreviewed' THEN 'auto-scored' ELSE ${skills.reviewStatus} END` }
|
||||||
: {}),
|
: {}),
|
||||||
|
// Clear stale state on successful re-index (skill is accessible again)
|
||||||
|
isStale: false,
|
||||||
|
staleSince: null,
|
||||||
|
staleCheckCount: 0,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
.returning();
|
.returning();
|
||||||
@@ -759,6 +763,44 @@ export const skillQueries = {
|
|||||||
await db.update(skills).set({ isBlocked: false }).where(eq(skills.id, id));
|
await db.update(skills).set({ isBlocked: false }).where(eq(skills.id, id));
|
||||||
},
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mark all skills from a repo as owner-claimed (exempt from duplicate hiding)
|
||||||
|
*/
|
||||||
|
setOwnerClaimed: async (db: DB, owner: string, repo: string) => {
|
||||||
|
await db.update(skills)
|
||||||
|
.set({ isOwnerClaimed: true })
|
||||||
|
.where(and(eq(skills.githubOwner, owner), eq(skills.githubRepo, repo)));
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Block all skills from a repo (owner requested repo-level removal)
|
||||||
|
*/
|
||||||
|
blockByRepo: async (db: DB, owner: string, repo: string) => {
|
||||||
|
await db.update(skills)
|
||||||
|
.set({ isBlocked: true })
|
||||||
|
.where(and(eq(skills.githubOwner, owner), eq(skills.githubRepo, repo)));
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Unblock all skills from a repo (owner re-enabled their repository)
|
||||||
|
*/
|
||||||
|
unblockByRepo: async (db: DB, owner: string, repo: string) => {
|
||||||
|
await db.update(skills)
|
||||||
|
.set({ isBlocked: false })
|
||||||
|
.where(and(eq(skills.githubOwner, owner), eq(skills.githubRepo, repo)));
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Count non-blocked skills in a repo
|
||||||
|
*/
|
||||||
|
countByRepo: async (db: DB, owner: string, repo: string): Promise<number> => {
|
||||||
|
const result = await db
|
||||||
|
.select({ count: sql<number>`count(*)::int` })
|
||||||
|
.from(skills)
|
||||||
|
.where(and(eq(skills.githubOwner, owner), eq(skills.githubRepo, repo), eq(skills.isBlocked, false)));
|
||||||
|
return result[0]?.count ?? 0;
|
||||||
|
},
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if a skill is blocked
|
* Check if a skill is blocked
|
||||||
*/
|
*/
|
||||||
@@ -771,6 +813,92 @@ export const skillQueries = {
|
|||||||
return result[0]?.isBlocked ?? false;
|
return result[0]?.isBlocked ?? false;
|
||||||
},
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Increment stale check count. After 3 consecutive 404s, mark skill as stale.
|
||||||
|
* Sets stale_since on first detection. Idempotent if skill doesn't exist.
|
||||||
|
*/
|
||||||
|
incrementStaleCheck: async (db: DB, id: string) => {
|
||||||
|
const STALE_THRESHOLD = 3;
|
||||||
|
|
||||||
|
const existing = await db
|
||||||
|
.select({
|
||||||
|
staleCheckCount: skills.staleCheckCount,
|
||||||
|
staleSince: skills.staleSince,
|
||||||
|
})
|
||||||
|
.from(skills)
|
||||||
|
.where(eq(skills.id, id))
|
||||||
|
.limit(1);
|
||||||
|
|
||||||
|
if (!existing[0]) return;
|
||||||
|
|
||||||
|
const newCount = (existing[0].staleCheckCount ?? 0) + 1;
|
||||||
|
const isNowStale = newCount >= STALE_THRESHOLD;
|
||||||
|
|
||||||
|
await db.update(skills).set({
|
||||||
|
staleCheckCount: newCount,
|
||||||
|
staleSince: existing[0].staleSince ?? new Date(),
|
||||||
|
isStale: isNowStale,
|
||||||
|
}).where(eq(skills.id, id));
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Clear stale state when a skill is successfully re-fetched from GitHub
|
||||||
|
*/
|
||||||
|
clearStaleState: async (db: DB, id: string) => {
|
||||||
|
await db.update(skills).set({
|
||||||
|
isStale: false,
|
||||||
|
staleSince: null,
|
||||||
|
staleCheckCount: 0,
|
||||||
|
}).where(eq(skills.id, id));
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get stale skills (for admin/reporting)
|
||||||
|
*/
|
||||||
|
getStaleSkills: async (db: DB, limit = 100, offset = 0) => {
|
||||||
|
return db
|
||||||
|
.select()
|
||||||
|
.from(skills)
|
||||||
|
.where(eq(skills.isStale, true))
|
||||||
|
.orderBy(desc(skills.staleSince))
|
||||||
|
.limit(limit)
|
||||||
|
.offset(offset);
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Count stale skills
|
||||||
|
*/
|
||||||
|
countStale: async (db: DB): Promise<number> => {
|
||||||
|
const result = await db
|
||||||
|
.select({ count: sql<number>`count(*)::int` })
|
||||||
|
.from(skills)
|
||||||
|
.where(eq(skills.isStale, true));
|
||||||
|
return result[0]?.count ?? 0;
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get skills to check for staleness (ordered by oldest scanned first)
|
||||||
|
*/
|
||||||
|
getSkillsToStaleCheck: async (db: DB, limit = 500) => {
|
||||||
|
return db
|
||||||
|
.select({
|
||||||
|
id: skills.id,
|
||||||
|
githubOwner: skills.githubOwner,
|
||||||
|
githubRepo: skills.githubRepo,
|
||||||
|
skillPath: skills.skillPath,
|
||||||
|
branch: skills.branch,
|
||||||
|
sourceFormat: skills.sourceFormat,
|
||||||
|
staleCheckCount: skills.staleCheckCount,
|
||||||
|
})
|
||||||
|
.from(skills)
|
||||||
|
.where(and(
|
||||||
|
eq(skills.isBlocked, false),
|
||||||
|
eq(skills.isStale, false),
|
||||||
|
))
|
||||||
|
.orderBy(asc(skills.lastScanned))
|
||||||
|
.limit(limit);
|
||||||
|
},
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get cached files for a skill
|
* Get cached files for a skill
|
||||||
* Returns null if cache doesn't exist or is stale (commitSha mismatch)
|
* Returns null if cache doesn't exist or is stale (commitSha mismatch)
|
||||||
@@ -1266,6 +1394,7 @@ export async function runPostCrawlCuration(db: DB): Promise<{ classified: number
|
|||||||
UPDATE skills s SET is_duplicate = true
|
UPDATE skills s SET is_duplicate = true
|
||||||
WHERE s.is_blocked = false
|
WHERE s.is_blocked = false
|
||||||
AND s.is_duplicate = false
|
AND s.is_duplicate = false
|
||||||
|
AND s.is_owner_claimed = false
|
||||||
AND s.content_hash IS NOT NULL
|
AND s.content_hash IS NOT NULL
|
||||||
AND EXISTS (
|
AND EXISTS (
|
||||||
SELECT 1 FROM skills s2
|
SELECT 1 FROM skills s2
|
||||||
@@ -1623,6 +1752,7 @@ export const discoveredRepoQueries = {
|
|||||||
conditions.push(sql`${discoveredRepos.lastScanned} IS NULL`);
|
conditions.push(sql`${discoveredRepos.lastScanned} IS NULL`);
|
||||||
}
|
}
|
||||||
conditions.push(eq(discoveredRepos.isArchived, false));
|
conditions.push(eq(discoveredRepos.isArchived, false));
|
||||||
|
conditions.push(eq(discoveredRepos.isBlocked, false));
|
||||||
|
|
||||||
return db
|
return db
|
||||||
.select()
|
.select()
|
||||||
@@ -1632,6 +1762,20 @@ export const discoveredRepoQueries = {
|
|||||||
.limit(limit);
|
.limit(limit);
|
||||||
},
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Block a repo from re-indexing (owner requested removal of all skills)
|
||||||
|
*/
|
||||||
|
blockRepo: async (db: DB, id: string) => {
|
||||||
|
await db.update(discoveredRepos).set({ isBlocked: true }).where(eq(discoveredRepos.id, id));
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Unblock a repo (owner re-enabled it after previously removing it)
|
||||||
|
*/
|
||||||
|
unblockRepo: async (db: DB, id: string) => {
|
||||||
|
await db.update(discoveredRepos).set({ isBlocked: false }).where(eq(discoveredRepos.id, id));
|
||||||
|
},
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Mark a repo as scanned
|
* Mark a repo as scanned
|
||||||
*/
|
*/
|
||||||
@@ -2511,6 +2655,7 @@ export const skillReviewQueries = {
|
|||||||
priorityReReview?: boolean;
|
priorityReReview?: boolean;
|
||||||
reReviewAll?: boolean;
|
reReviewAll?: boolean;
|
||||||
ownerLimit?: number;
|
ownerLimit?: number;
|
||||||
|
currentReviewVersion?: number;
|
||||||
} = {}
|
} = {}
|
||||||
) => {
|
) => {
|
||||||
const {
|
const {
|
||||||
@@ -2521,6 +2666,7 @@ export const skillReviewQueries = {
|
|||||||
priorityReReview = false,
|
priorityReReview = false,
|
||||||
reReviewAll = false,
|
reReviewAll = false,
|
||||||
ownerLimit = 0,
|
ownerLimit = 0,
|
||||||
|
currentReviewVersion = 0,
|
||||||
} = options;
|
} = options;
|
||||||
|
|
||||||
const conditions = [
|
const conditions = [
|
||||||
@@ -2539,6 +2685,16 @@ export const skillReviewQueries = {
|
|||||||
conditions.push(
|
conditions.push(
|
||||||
sql`${skills.reviewStatus} IN ('ai-reviewed', 'needs-re-review', 'auto-scored')`
|
sql`${skills.reviewStatus} IN ('ai-reviewed', 'needs-re-review', 'auto-scored')`
|
||||||
);
|
);
|
||||||
|
// Skip skills already reviewed at the current version
|
||||||
|
if (currentReviewVersion > 0) {
|
||||||
|
conditions.push(
|
||||||
|
sql`NOT EXISTS (
|
||||||
|
SELECT 1 FROM skill_reviews sr
|
||||||
|
WHERE sr.skill_id = ${skills.id}
|
||||||
|
AND sr.review_version >= ${currentReviewVersion}
|
||||||
|
)`
|
||||||
|
);
|
||||||
|
}
|
||||||
} else if (priorityReReview) {
|
} else if (priorityReReview) {
|
||||||
conditions.push(eq(skills.reviewStatus, 'needs-re-review'));
|
conditions.push(eq(skills.reviewStatus, 'needs-re-review'));
|
||||||
} else {
|
} else {
|
||||||
@@ -2570,6 +2726,12 @@ export const skillReviewQueries = {
|
|||||||
branch: skills.branch,
|
branch: skills.branch,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Re-review-all: prioritize already-reviewed skills first, then auto-scored
|
||||||
|
// Use sql.raw() for the ORDER BY to avoid parameterization issues in db.execute()
|
||||||
|
const orderBySql = reReviewAll
|
||||||
|
? sql`CASE WHEN review_status IN ('ai-reviewed', 'needs-re-review') THEN 0 ELSE 1 END, quality_score DESC NULLS LAST, github_stars DESC NULLS LAST`
|
||||||
|
: sql`quality_score DESC NULLS LAST, github_stars DESC NULLS LAST`;
|
||||||
|
|
||||||
// Owner-capped batch: limit skills per github_owner for diversity
|
// Owner-capped batch: limit skills per github_owner for diversity
|
||||||
if (ownerLimit > 0) {
|
if (ownerLimit > 0) {
|
||||||
const whereClause = and(...conditions);
|
const whereClause = and(...conditions);
|
||||||
@@ -2580,7 +2742,7 @@ export const skillReviewQueries = {
|
|||||||
github_owner, github_repo, skill_path, branch,
|
github_owner, github_repo, skill_path, branch,
|
||||||
ROW_NUMBER() OVER (
|
ROW_NUMBER() OVER (
|
||||||
PARTITION BY github_owner
|
PARTITION BY github_owner
|
||||||
ORDER BY quality_score DESC NULLS LAST, github_stars DESC NULLS LAST
|
ORDER BY ${orderBySql}
|
||||||
) AS owner_rank
|
) AS owner_rank
|
||||||
FROM skills
|
FROM skills
|
||||||
WHERE ${whereClause}
|
WHERE ${whereClause}
|
||||||
@@ -2593,18 +2755,23 @@ export const skillReviewQueries = {
|
|||||||
skill_path AS "skillPath", branch
|
skill_path AS "skillPath", branch
|
||||||
FROM ranked
|
FROM ranked
|
||||||
WHERE owner_rank <= ${ownerLimit}
|
WHERE owner_rank <= ${ownerLimit}
|
||||||
ORDER BY quality_score DESC NULLS LAST, github_stars DESC NULLS LAST
|
ORDER BY ${orderBySql}
|
||||||
LIMIT ${batchSize}
|
LIMIT ${batchSize}
|
||||||
OFFSET ${offset}
|
OFFSET ${offset}
|
||||||
`);
|
`);
|
||||||
return [...results];
|
return [...results];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Build order: re-review priority first (if applicable), then quality
|
||||||
|
const orderClauses = reReviewAll
|
||||||
|
? [sql`CASE WHEN ${skills.reviewStatus} IN ('ai-reviewed', 'needs-re-review') THEN 0 ELSE 1 END`, desc(skills.qualityScore), desc(skills.githubStars)]
|
||||||
|
: [desc(skills.qualityScore), desc(skills.githubStars)];
|
||||||
|
|
||||||
return db
|
return db
|
||||||
.select(selectFields)
|
.select(selectFields)
|
||||||
.from(skills)
|
.from(skills)
|
||||||
.where(and(...conditions))
|
.where(and(...conditions))
|
||||||
.orderBy(desc(skills.qualityScore), desc(skills.githubStars))
|
.orderBy(...orderClauses)
|
||||||
.limit(batchSize)
|
.limit(batchSize)
|
||||||
.offset(offset);
|
.offset(offset);
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -67,6 +67,9 @@ export const skills = pgTable(
|
|||||||
isFeatured: boolean('is_featured').default(false),
|
isFeatured: boolean('is_featured').default(false),
|
||||||
isBlocked: boolean('is_blocked').default(false), // Blocked from re-indexing (owner requested removal)
|
isBlocked: boolean('is_blocked').default(false), // Blocked from re-indexing (owner requested removal)
|
||||||
isDeprecated: boolean('is_deprecated').default(false), // Auto-detected from raw_content (DEPRECATED/ARCHIVED markers)
|
isDeprecated: boolean('is_deprecated').default(false), // Auto-detected from raw_content (DEPRECATED/ARCHIVED markers)
|
||||||
|
isStale: boolean('is_stale').default(false), // Skill files no longer accessible on GitHub (confirmed after 3 consecutive 404s)
|
||||||
|
staleSince: timestamp('stale_since'), // When staleness was first detected
|
||||||
|
staleCheckCount: integer('stale_check_count').default(0), // Consecutive 404 count (threshold: 3)
|
||||||
lastScanned: timestamp('last_scanned'),
|
lastScanned: timestamp('last_scanned'),
|
||||||
|
|
||||||
// Curation (populated by batch scripts, not crawler)
|
// Curation (populated by batch scripts, not crawler)
|
||||||
@@ -79,6 +82,7 @@ export const skills = pgTable(
|
|||||||
}>(),
|
}>(),
|
||||||
skillType: text('skill_type').$type<'standalone' | 'project-bound' | 'collection' | 'aggregator'>(),
|
skillType: text('skill_type').$type<'standalone' | 'project-bound' | 'collection' | 'aggregator'>(),
|
||||||
isDuplicate: boolean('is_duplicate').default(false),
|
isDuplicate: boolean('is_duplicate').default(false),
|
||||||
|
isOwnerClaimed: boolean('is_owner_claimed').default(false), // Owner explicitly submitted via Add Skill — exempt from duplicate hiding
|
||||||
canonicalSkillId: text('canonical_skill_id'), // points to the "original" if this is a duplicate
|
canonicalSkillId: text('canonical_skill_id'), // points to the "original" if this is a duplicate
|
||||||
repoSkillCount: integer('repo_skill_count'), // cached count of skills in same repo
|
repoSkillCount: integer('repo_skill_count'), // cached count of skills in same repo
|
||||||
reviewStatus: text('review_status').default('unreviewed'), // unreviewed → auto-scored → ai-reviewed → verified (or needs-re-review)
|
reviewStatus: text('review_status').default('unreviewed'), // unreviewed → auto-scored → ai-reviewed → verified (or needs-re-review)
|
||||||
@@ -124,6 +128,7 @@ export const skills = pgTable(
|
|||||||
skillTypeIdx: index('idx_skills_type').on(table.skillType),
|
skillTypeIdx: index('idx_skills_type').on(table.skillType),
|
||||||
duplicateIdx: index('idx_skills_duplicate').on(table.isDuplicate),
|
duplicateIdx: index('idx_skills_duplicate').on(table.isDuplicate),
|
||||||
contentHashIdx: index('idx_skills_content_hash').on(table.contentHash),
|
contentHashIdx: index('idx_skills_content_hash').on(table.contentHash),
|
||||||
|
staleIdx: index('idx_skills_stale').on(table.isStale),
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -303,6 +308,7 @@ export const discoveredRepos = pgTable(
|
|||||||
githubForks: integer('github_forks').default(0),
|
githubForks: integer('github_forks').default(0),
|
||||||
defaultBranch: text('default_branch').default('main'),
|
defaultBranch: text('default_branch').default('main'),
|
||||||
isArchived: boolean('is_archived').default(false),
|
isArchived: boolean('is_archived').default(false),
|
||||||
|
isBlocked: boolean('is_blocked').default(false), // Owner requested removal — skip re-indexing
|
||||||
scanError: text('scan_error'), // Last scan error if any
|
scanError: text('scan_error'), // Last scan error if any
|
||||||
createdAt: timestamp('created_at').defaultNow().notNull(),
|
createdAt: timestamp('created_at').defaultNow().notNull(),
|
||||||
updatedAt: timestamp('updated_at').defaultNow().notNull(),
|
updatedAt: timestamp('updated_at').defaultNow().notNull(),
|
||||||
|
|||||||
@@ -164,6 +164,7 @@ CREATE TABLE IF NOT EXISTS discovered_repos (
|
|||||||
github_forks INTEGER DEFAULT 0,
|
github_forks INTEGER DEFAULT 0,
|
||||||
default_branch TEXT DEFAULT 'main',
|
default_branch TEXT DEFAULT 'main',
|
||||||
is_archived BOOLEAN DEFAULT FALSE,
|
is_archived BOOLEAN DEFAULT FALSE,
|
||||||
|
is_blocked BOOLEAN NOT NULL DEFAULT FALSE,
|
||||||
scan_error TEXT,
|
scan_error TEXT,
|
||||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||||
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL
|
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL
|
||||||
@@ -471,13 +472,17 @@ ALTER TABLE skills ADD COLUMN IF NOT EXISTS quality_score INTEGER;
|
|||||||
ALTER TABLE skills ADD COLUMN IF NOT EXISTS quality_details JSONB;
|
ALTER TABLE skills ADD COLUMN IF NOT EXISTS quality_details JSONB;
|
||||||
ALTER TABLE skills ADD COLUMN IF NOT EXISTS skill_type TEXT; -- 'standalone', 'project-bound', 'collection', 'aggregator'
|
ALTER TABLE skills ADD COLUMN IF NOT EXISTS skill_type TEXT; -- 'standalone', 'project-bound', 'collection', 'aggregator'
|
||||||
ALTER TABLE skills ADD COLUMN IF NOT EXISTS is_duplicate BOOLEAN DEFAULT FALSE;
|
ALTER TABLE skills ADD COLUMN IF NOT EXISTS is_duplicate BOOLEAN DEFAULT FALSE;
|
||||||
|
ALTER TABLE skills ADD COLUMN IF NOT EXISTS is_owner_claimed BOOLEAN NOT NULL DEFAULT FALSE;
|
||||||
ALTER TABLE skills ADD COLUMN IF NOT EXISTS canonical_skill_id TEXT; -- points to original if duplicate
|
ALTER TABLE skills ADD COLUMN IF NOT EXISTS canonical_skill_id TEXT; -- points to original if duplicate
|
||||||
ALTER TABLE skills ADD COLUMN IF NOT EXISTS repo_skill_count INTEGER; -- cached count of skills in repo
|
ALTER TABLE skills ADD COLUMN IF NOT EXISTS repo_skill_count INTEGER; -- cached count of skills in repo
|
||||||
|
ALTER TABLE discovered_repos ADD COLUMN IF NOT EXISTS is_blocked BOOLEAN NOT NULL DEFAULT FALSE;
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_skills_quality ON skills(quality_score DESC NULLS LAST);
|
CREATE INDEX IF NOT EXISTS idx_skills_quality ON skills(quality_score DESC NULLS LAST);
|
||||||
CREATE INDEX IF NOT EXISTS idx_skills_type ON skills(skill_type);
|
CREATE INDEX IF NOT EXISTS idx_skills_type ON skills(skill_type);
|
||||||
CREATE INDEX IF NOT EXISTS idx_skills_duplicate ON skills(is_duplicate);
|
CREATE INDEX IF NOT EXISTS idx_skills_duplicate ON skills(is_duplicate);
|
||||||
CREATE INDEX IF NOT EXISTS idx_skills_content_hash ON skills(content_hash);
|
CREATE INDEX IF NOT EXISTS idx_skills_content_hash ON skills(content_hash);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_skills_owner_claimed ON skills(is_owner_claimed) WHERE is_owner_claimed = TRUE;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_discovered_repos_blocked ON discovered_repos(is_blocked) WHERE is_blocked = TRUE;
|
||||||
|
|
||||||
-- Review pipeline columns (Phase 4+5 - February 2026)
|
-- Review pipeline columns (Phase 4+5 - February 2026)
|
||||||
ALTER TABLE skills ADD COLUMN IF NOT EXISTS review_status TEXT DEFAULT 'unreviewed';
|
ALTER TABLE skills ADD COLUMN IF NOT EXISTS review_status TEXT DEFAULT 'unreviewed';
|
||||||
@@ -557,6 +562,12 @@ CREATE INDEX IF NOT EXISTS idx_reviews_skill ON skill_reviews(skill_id);
|
|||||||
CREATE INDEX IF NOT EXISTS idx_reviews_score ON skill_reviews(ai_score);
|
CREATE INDEX IF NOT EXISTS idx_reviews_score ON skill_reviews(ai_score);
|
||||||
CREATE INDEX IF NOT EXISTS idx_reviews_blog ON skill_reviews(blog_worthy) WHERE blog_worthy = true;
|
CREATE INDEX IF NOT EXISTS idx_reviews_blog ON skill_reviews(blog_worthy) WHERE blog_worthy = true;
|
||||||
|
|
||||||
|
-- Stale skill detection (March 2026)
|
||||||
|
ALTER TABLE skills ADD COLUMN IF NOT EXISTS is_stale BOOLEAN DEFAULT FALSE;
|
||||||
|
ALTER TABLE skills ADD COLUMN IF NOT EXISTS stale_since TIMESTAMP WITH TIME ZONE;
|
||||||
|
ALTER TABLE skills ADD COLUMN IF NOT EXISTS stale_check_count INTEGER DEFAULT 0;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_skills_stale ON skills(is_stale) WHERE is_stale = TRUE;
|
||||||
|
|
||||||
-- Grant permissions
|
-- Grant permissions
|
||||||
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO postgres;
|
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO postgres;
|
||||||
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO postgres;
|
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO postgres;
|
||||||
|
|||||||
@@ -7,7 +7,7 @@
|
|||||||
import { INSTRUCTION_FILE_PATTERNS, type SourceFormat } from 'skillhub-core';
|
import { INSTRUCTION_FILE_PATTERNS, type SourceFormat } from 'skillhub-core';
|
||||||
import { scheduleFullCrawl, scheduleIncrementalCrawl, getQueueStats, getQueue } from './queue.js';
|
import { scheduleFullCrawl, scheduleIncrementalCrawl, getQueueStats, getQueue } from './queue.js';
|
||||||
import { syncAllSkillsToMeilisearch, checkMeilisearchHealth } from './meilisearch-sync.js';
|
import { syncAllSkillsToMeilisearch, checkMeilisearchHealth } from './meilisearch-sync.js';
|
||||||
import { createDb, skillQueries, categoryQueries, discoveredRepoQueries, awesomeListQueries, addRequestQueries, userQueries, sql } from '@skillhub/db';
|
import { createDb, skillQueries, categoryQueries, discoveredRepoQueries, awesomeListQueries, addRequestQueries, userQueries, skills, sql } from '@skillhub/db';
|
||||||
import { createStrategyOrchestrator, createDeepScanCrawler, createAwesomeListCrawler, createPopularReposCrawler, createCommitsSearchCrawler } from './strategies/index.js';
|
import { createStrategyOrchestrator, createDeepScanCrawler, createAwesomeListCrawler, createPopularReposCrawler, createCommitsSearchCrawler } from './strategies/index.js';
|
||||||
import { createCrawler } from './crawler.js';
|
import { createCrawler } from './crawler.js';
|
||||||
import { indexSkill } from './skill-indexer.js';
|
import { indexSkill } from './skill-indexer.js';
|
||||||
@@ -870,6 +870,9 @@ async function main() {
|
|||||||
status: 'indexed',
|
status: 'indexed',
|
||||||
indexedSkillId: allSkillIds.join(','),
|
indexedSkillId: allSkillIds.join(','),
|
||||||
});
|
});
|
||||||
|
// Mark all skills from this repo as owner-claimed so they're
|
||||||
|
// exempt from duplicate hiding on the owner page and browse pages
|
||||||
|
await skillQueries.setOwnerClaimed(addDb, owner, repo);
|
||||||
const newCount = indexedSkillIds.length;
|
const newCount = indexedSkillIds.length;
|
||||||
const existingCount = existingSkillIds.length;
|
const existingCount = existingSkillIds.length;
|
||||||
if (newCount > 0 && existingCount > 0) {
|
if (newCount > 0 && existingCount > 0) {
|
||||||
@@ -926,6 +929,170 @@ async function main() {
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case 'add-repo': {
|
||||||
|
const input = process.argv[3];
|
||||||
|
if (!input) {
|
||||||
|
console.error('Usage: node dist/crawl.js add-repo <owner/repo or https://github.com/owner/repo>');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
let arOwner: string;
|
||||||
|
let arRepo: string;
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (input.startsWith('https://') || input.startsWith('http://')) {
|
||||||
|
const arUrl = new URL(input);
|
||||||
|
const arParts = arUrl.pathname.split('/').filter(Boolean);
|
||||||
|
arOwner = arParts[0];
|
||||||
|
arRepo = arParts[1];
|
||||||
|
} else {
|
||||||
|
const arParts = input.split('/').filter(Boolean);
|
||||||
|
arOwner = arParts[0];
|
||||||
|
arRepo = arParts[1];
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
console.error('Could not parse owner/repo from input:', input);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!arOwner! || !arRepo!) {
|
||||||
|
console.error('Could not parse owner/repo from input:', input);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`Validating ${arOwner}/${arRepo} on GitHub...`);
|
||||||
|
const arGhRes = await fetch(`https://api.github.com/repos/${arOwner}/${arRepo}`, {
|
||||||
|
headers: {
|
||||||
|
Authorization: `token ${process.env.GITHUB_TOKEN}`,
|
||||||
|
'User-Agent': 'SkillHub',
|
||||||
|
Accept: 'application/vnd.github.v3+json',
|
||||||
|
},
|
||||||
|
signal: AbortSignal.timeout(10000),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!arGhRes.ok) {
|
||||||
|
console.error(`Repo ${arOwner}/${arRepo} not found on GitHub (HTTP ${arGhRes.status})`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const arGhData = await arGhRes.json() as {
|
||||||
|
stargazers_count: number;
|
||||||
|
forks_count: number;
|
||||||
|
default_branch: string;
|
||||||
|
archived: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
const arDb = createDb(process.env.DATABASE_URL);
|
||||||
|
await discoveredRepoQueries.upsert(arDb, {
|
||||||
|
id: `${arOwner}/${arRepo}`,
|
||||||
|
owner: arOwner,
|
||||||
|
repo: arRepo,
|
||||||
|
discoveredVia: 'manual',
|
||||||
|
sourceUrl: `https://github.com/${arOwner}/${arRepo}`,
|
||||||
|
githubStars: arGhData.stargazers_count ?? 0,
|
||||||
|
githubForks: arGhData.forks_count ?? 0,
|
||||||
|
defaultBranch: arGhData.default_branch ?? 'main',
|
||||||
|
isArchived: arGhData.archived ?? false,
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(`✅ Added ${arOwner}/${arRepo} to discovered_repos.`);
|
||||||
|
console.log(` Run 'deep-scan' to index skills from this repository.`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
case 'stale-check': {
|
||||||
|
console.log('Checking for stale skills (removed from GitHub)...\n');
|
||||||
|
const staleDb = createDb(process.env.DATABASE_URL);
|
||||||
|
const staleCrawler = createCrawler();
|
||||||
|
|
||||||
|
// Parse --limit=N option
|
||||||
|
const staleLimitArg = process.argv.find(a => a.startsWith('--limit='));
|
||||||
|
const staleLimit = staleLimitArg ? parseInt(staleLimitArg.split('=')[1]) : 500;
|
||||||
|
|
||||||
|
// Check GitHub rate limit
|
||||||
|
try {
|
||||||
|
const rateLimitInfo = await staleCrawler.getRateLimitStatus();
|
||||||
|
console.log(`GitHub API: ${rateLimitInfo.remaining}/${rateLimitInfo.limit} requests remaining`);
|
||||||
|
if (rateLimitInfo.remaining < 50) {
|
||||||
|
console.log('Not enough API quota for stale checking. Try again later.');
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
console.log('');
|
||||||
|
} catch {
|
||||||
|
console.log('Could not check GitHub rate limit. Proceeding anyway...\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get skills to check (oldest scanned first)
|
||||||
|
const skillsToCheck = await skillQueries.getSkillsToStaleCheck(staleDb, staleLimit);
|
||||||
|
|
||||||
|
console.log(`Checking ${skillsToCheck.length} skills...\n`);
|
||||||
|
|
||||||
|
let staleChecked = 0;
|
||||||
|
let stillAlive = 0;
|
||||||
|
let newlyStale = 0;
|
||||||
|
let staleErrors = 0;
|
||||||
|
|
||||||
|
for (const skill of skillsToCheck) {
|
||||||
|
// Rate limit check every 50 skills
|
||||||
|
if (staleChecked > 0 && staleChecked % 50 === 0) {
|
||||||
|
try {
|
||||||
|
const midCheck = await staleCrawler.getRateLimitStatus();
|
||||||
|
if (midCheck.remaining < 20) {
|
||||||
|
console.log(`\n API quota low (${midCheck.remaining} remaining). Stopping.`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Continue if rate limit check fails
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const sourceFormat = skill.sourceFormat || 'skill.md';
|
||||||
|
const pattern = INSTRUCTION_FILE_PATTERNS.find(p => p.format === sourceFormat);
|
||||||
|
const filename = pattern?.filename || 'SKILL.md';
|
||||||
|
const filePath = skill.skillPath === '.' ? filename : `${skill.skillPath}/${filename}`;
|
||||||
|
|
||||||
|
const exists = await staleCrawler.checkFileExists(
|
||||||
|
skill.githubOwner,
|
||||||
|
skill.githubRepo,
|
||||||
|
filePath,
|
||||||
|
skill.branch || 'main'
|
||||||
|
);
|
||||||
|
|
||||||
|
if (exists) {
|
||||||
|
stillAlive++;
|
||||||
|
// Clear any partial stale state
|
||||||
|
if ((skill.staleCheckCount ?? 0) > 0) {
|
||||||
|
await skillQueries.clearStaleState(staleDb, skill.id);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
await skillQueries.incrementStaleCheck(staleDb, skill.id);
|
||||||
|
newlyStale++;
|
||||||
|
console.log(` STALE: ${skill.id}`);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
staleErrors++;
|
||||||
|
}
|
||||||
|
staleChecked++;
|
||||||
|
|
||||||
|
// Progress every 100
|
||||||
|
if (staleChecked % 100 === 0) {
|
||||||
|
console.log(` Progress: ${staleChecked}/${skillsToCheck.length} checked`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`\nStale check complete:`);
|
||||||
|
console.log(` Checked: ${staleChecked}`);
|
||||||
|
console.log(` Still alive: ${stillAlive}`);
|
||||||
|
console.log(` Stale detected: ${newlyStale}`);
|
||||||
|
console.log(` Errors: ${staleErrors}`);
|
||||||
|
|
||||||
|
// Show total stale count
|
||||||
|
const totalStale = await skillQueries.countStale(staleDb);
|
||||||
|
console.log(` Total stale skills in DB: ${totalStale}`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
default:
|
default:
|
||||||
console.log('Usage: pnpm crawl <command>\n');
|
console.log('Usage: pnpm crawl <command>\n');
|
||||||
@@ -944,6 +1111,7 @@ async function main() {
|
|||||||
console.log(' sync-meili - Sync all skills from database to Meilisearch');
|
console.log(' sync-meili - Sync all skills from database to Meilisearch');
|
||||||
console.log(' recategorize - Re-categorize all skills with 23 categories (alias: link-categories)');
|
console.log(' recategorize - Re-categorize all skills with 23 categories (alias: link-categories)');
|
||||||
console.log(' process-add-requests - Process pending add requests (auto-index skills)');
|
console.log(' process-add-requests - Process pending add requests (auto-index skills)');
|
||||||
|
console.log(' stale-check - Check for stale skills removed from GitHub (--limit=N)');
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -215,7 +215,7 @@ export class GitHubCrawler {
|
|||||||
/**
|
/**
|
||||||
* Check if a file exists in a repository
|
* Check if a file exists in a repository
|
||||||
*/
|
*/
|
||||||
private async checkFileExists(owner: string, repo: string, path: string, ref: string): Promise<boolean> {
|
async checkFileExists(owner: string, repo: string, path: string, ref: string): Promise<boolean> {
|
||||||
try {
|
try {
|
||||||
const { octokit, token } = await this.getOctokit();
|
const { octokit, token } = await this.getOctokit();
|
||||||
const response = await octokit.repos.getContent({
|
const response = await octokit.repos.getContent({
|
||||||
|
|||||||
@@ -33,7 +33,28 @@ export async function indexSkill(
|
|||||||
|
|
||||||
// Fetch skill content
|
// Fetch skill content
|
||||||
console.log(`Fetching ${source.owner}/${source.repo}/${source.path} [${sourceFormat}]...`);
|
console.log(`Fetching ${source.owner}/${source.repo}/${source.path} [${sourceFormat}]...`);
|
||||||
const content = await crawler.fetchSkillContent(source);
|
let content;
|
||||||
|
try {
|
||||||
|
content = await crawler.fetchSkillContent(source);
|
||||||
|
} catch (error) {
|
||||||
|
const errorMsg = error instanceof Error ? error.message : String(error);
|
||||||
|
|
||||||
|
// Detect 404 (skill removed from GitHub) — increment stale check
|
||||||
|
if (errorMsg.includes('not found') || errorMsg.includes('Not Found') || errorMsg.includes('404')) {
|
||||||
|
const skillName = source.path.split('/').pop() || 'skill';
|
||||||
|
const formatSuffix = sourceFormat !== 'skill.md' ? `~${sourceFormat.replace('.', '')}` : '';
|
||||||
|
const skillId = `${source.owner}/${source.repo}/${skillName}${formatSuffix}`;
|
||||||
|
|
||||||
|
await skillQueries.incrementStaleCheck(database, skillId).catch((err) => {
|
||||||
|
console.warn(` -> Failed to increment stale check for ${skillId}:`, err);
|
||||||
|
});
|
||||||
|
console.log(` -> ${skillId}: GitHub 404 (stale check incremented)`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// For non-404 errors, rethrow
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
// Analyze the skill with format awareness
|
// Analyze the skill with format awareness
|
||||||
const analysis = await analyzer.analyze(content, sourceFormat);
|
const analysis = await analyzer.analyze(content, sourceFormat);
|
||||||
|
|||||||
Reference in New Issue
Block a user