diff --git a/README.md b/README.md index 367804b..6976191 100644 --- a/README.md +++ b/README.md @@ -184,18 +184,6 @@ Always review skill source code before installing. --- -## Roadmap - -- [x] Web marketplace with growing skill catalog -- [x] CLI tool -- [x] Multi-platform support -- [x] Security scanning -- [ ] Claude Code MCP plugin -- [ ] AI-powered recommendations -- [ ] Enterprise features - ---- - ## License MIT - See [LICENSE](./LICENSE) for details. diff --git a/apps/web/app/[locale]/about/page.tsx b/apps/web/app/[locale]/about/page.tsx index a671700..1fbfa14 100644 --- a/apps/web/app/[locale]/about/page.tsx +++ b/apps/web/app/[locale]/about/page.tsx @@ -2,6 +2,20 @@ import { getTranslations, setRequestLocale } from 'next-intl/server'; import { Header } from '@/components/Header'; import { Footer } from '@/components/Footer'; import { Code2, Globe, Shield } from 'lucide-react'; +import { getPageAlternates } from '@/lib/seo'; + + + +export async function generateMetadata({ + params, +}: { + params: Promise<{ locale: string }>; +}) { + const { locale } = await params; + return { + alternates: getPageAlternates(locale, '/about'), + }; +} export default async function AboutPage({ params, diff --git a/apps/web/app/[locale]/attribution/page.tsx b/apps/web/app/[locale]/attribution/page.tsx index bcdb2ee..b38e471 100644 --- a/apps/web/app/[locale]/attribution/page.tsx +++ b/apps/web/app/[locale]/attribution/page.tsx @@ -3,6 +3,8 @@ import Link from 'next/link'; import { Header } from '@/components/Header'; import { Footer } from '@/components/Footer'; import { Github, Heart, Users, Code, GitFork, ExternalLink, Database, Clock } from 'lucide-react'; +import { getPageAlternates } from '@/lib/seo'; + export const dynamic = 'force-dynamic'; @@ -65,6 +67,18 @@ function formatLicenseName(license: string, locale: string): string { return names ? names[locale as 'en' | 'fa'] || names.en : license; } + +export async function generateMetadata({ + params, +}: { + params: Promise<{ locale: string }>; +}) { + const { locale } = await params; + return { + alternates: getPageAlternates(locale, '/attribution'), + }; +} + export default async function AttributionPage({ params, }: { diff --git a/apps/web/app/[locale]/browse/page.tsx b/apps/web/app/[locale]/browse/page.tsx index c8ada34..e9f2372 100644 --- a/apps/web/app/[locale]/browse/page.tsx +++ b/apps/web/app/[locale]/browse/page.tsx @@ -5,6 +5,8 @@ import { createDb, skillQueries, categoryQueries } from '@skillhub/db'; import { BrowseFilters, SearchBar, Pagination, ActiveFilters, EmptyState } from '@/components/BrowseFilters'; import { SkillCard } from '@/components/SkillCard'; import { toPersianNumber } from '@/lib/format-number'; +import { getPageAlternates } from '@/lib/seo'; + // Force dynamic rendering to fetch fresh data from database export const dynamic = 'force-dynamic'; @@ -79,6 +81,18 @@ async function getSkills(params: { } } + +export async function generateMetadata({ + params, +}: { + params: Promise<{ locale: string }>; +}) { + const { locale } = await params; + return { + alternates: getPageAlternates(locale, '/browse'), + }; +} + export default async function BrowsePage({ params, searchParams }: BrowsePageProps) { const { locale } = await params; setRequestLocale(locale); diff --git a/apps/web/app/[locale]/categories/page.tsx b/apps/web/app/[locale]/categories/page.tsx index 60fab49..fcc1ff2 100644 --- a/apps/web/app/[locale]/categories/page.tsx +++ b/apps/web/app/[locale]/categories/page.tsx @@ -31,6 +31,8 @@ import { } from 'lucide-react'; import { createDb, categoryQueries } from '@skillhub/db'; import { formatNumber } from '@/lib/format-number'; +import { getPageAlternates } from '@/lib/seo'; + // Force dynamic rendering to fetch fresh data from database export const dynamic = 'force-dynamic'; @@ -96,6 +98,18 @@ async function getHierarchicalCategories() { } } + +export async function generateMetadata({ + params, +}: { + params: Promise<{ locale: string }>; +}) { + const { locale } = await params; + return { + alternates: getPageAlternates(locale, '/categories'), + }; +} + export default async function CategoriesPage({ params, }: { diff --git a/apps/web/app/[locale]/claim/page.tsx b/apps/web/app/[locale]/claim/page.tsx index a3fb861..8a6cfe4 100644 --- a/apps/web/app/[locale]/claim/page.tsx +++ b/apps/web/app/[locale]/claim/page.tsx @@ -2,9 +2,23 @@ import { getTranslations, setRequestLocale } from 'next-intl/server'; import { Header } from '@/components/Header'; import { Footer } from '@/components/Footer'; import { ClaimForm } from '@/components/ClaimForm'; +import { getPageAlternates } from '@/lib/seo'; + export const dynamic = 'force-dynamic'; + +export async function generateMetadata({ + params, +}: { + params: Promise<{ locale: string }>; +}) { + const { locale } = await params; + return { + alternates: getPageAlternates(locale, '/claim'), + }; +} + export default async function ClaimPage({ params, }: { diff --git a/apps/web/app/[locale]/claude-plugin/page.tsx b/apps/web/app/[locale]/claude-plugin/page.tsx index 4279253..ff02f38 100644 --- a/apps/web/app/[locale]/claude-plugin/page.tsx +++ b/apps/web/app/[locale]/claude-plugin/page.tsx @@ -1,4 +1,5 @@ import { getTranslations, setRequestLocale } from 'next-intl/server'; +import { getPageAlternates } from '@/lib/seo'; import Link from 'next/link'; import type { Metadata } from 'next'; import { @@ -31,6 +32,7 @@ export async function generateMetadata({ return { title: t('metadata.title'), description: t('metadata.description'), + alternates: getPageAlternates(locale, '/claude-plugin'), openGraph: { title: t('metadata.title'), description: t('metadata.description'), diff --git a/apps/web/app/[locale]/contact/page.tsx b/apps/web/app/[locale]/contact/page.tsx index 804d671..f06132b 100644 --- a/apps/web/app/[locale]/contact/page.tsx +++ b/apps/web/app/[locale]/contact/page.tsx @@ -1,4 +1,18 @@ import { redirect } from 'next/navigation'; +import { getPageAlternates } from '@/lib/seo'; + + + +export async function generateMetadata({ + params, +}: { + params: Promise<{ locale: string }>; +}) { + const { locale } = await params; + return { + alternates: getPageAlternates(locale, '/contact'), + }; +} export default async function ContactPage({ params, diff --git a/apps/web/app/[locale]/docs/api/page.tsx b/apps/web/app/[locale]/docs/api/page.tsx index b4b7b29..a912b72 100644 --- a/apps/web/app/[locale]/docs/api/page.tsx +++ b/apps/web/app/[locale]/docs/api/page.tsx @@ -5,6 +5,20 @@ import { ApiEndpointSection } from '@/components/ApiEndpointSection'; import type { EndpointDef } from '@/components/ApiEndpointSection'; import Link from 'next/link'; import { ArrowLeft, ArrowRight, Search, FileCode, Users, Compass, Mail } from 'lucide-react'; +import { getPageAlternates } from '@/lib/seo'; + + + +export async function generateMetadata({ + params, +}: { + params: Promise<{ locale: string }>; +}) { + const { locale } = await params; + return { + alternates: getPageAlternates(locale, '/docs/api'), + }; +} export default async function ApiDocsPage({ params, diff --git a/apps/web/app/[locale]/docs/cli/page.tsx b/apps/web/app/[locale]/docs/cli/page.tsx index 6382ee6..99a7ea9 100644 --- a/apps/web/app/[locale]/docs/cli/page.tsx +++ b/apps/web/app/[locale]/docs/cli/page.tsx @@ -3,6 +3,20 @@ import { Header } from '@/components/Header'; import { Footer } from '@/components/Footer'; import Link from 'next/link'; import { ArrowLeft, ArrowRight } from 'lucide-react'; +import { getPageAlternates } from '@/lib/seo'; + + + +export async function generateMetadata({ + params, +}: { + params: Promise<{ locale: string }>; +}) { + const { locale } = await params; + return { + alternates: getPageAlternates(locale, '/docs/cli'), + }; +} export default async function CliDocsPage({ params, diff --git a/apps/web/app/[locale]/docs/getting-started/page.tsx b/apps/web/app/[locale]/docs/getting-started/page.tsx index b606de0..0753efc 100644 --- a/apps/web/app/[locale]/docs/getting-started/page.tsx +++ b/apps/web/app/[locale]/docs/getting-started/page.tsx @@ -6,6 +6,8 @@ import Link from 'next/link'; import { ArrowLeft, ArrowRight } from 'lucide-react'; import { createDb, skills, sql } from '@skillhub/db'; import { formatPromptSkillCount } from '@/lib/format-number'; +import { getPageAlternates } from '@/lib/seo'; + export const dynamic = 'force-dynamic'; @@ -15,7 +17,7 @@ async function getSkillCount(): Promise { const result = await db .select({ count: sql`count(*)::int` }) .from(skills) - .where(sql`${skills.isDuplicate} = false AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`); + .where(sql`${skills.isDuplicate} = false`); return formatPromptSkillCount(result[0]?.count ?? 16000); } catch { return '16,000+'; @@ -56,6 +58,18 @@ Search for unfamiliar tech or complex tasks. Only install "Pass" security status }; } + +export async function generateMetadata({ + params, +}: { + params: Promise<{ locale: string }>; +}) { + const { locale } = await params; + return { + alternates: getPageAlternates(locale, '/docs/getting-started'), + }; +} + export default async function GettingStartedPage({ params, }: { diff --git a/apps/web/app/[locale]/docs/page.tsx b/apps/web/app/[locale]/docs/page.tsx index 4818113..00d1502 100644 --- a/apps/web/app/[locale]/docs/page.tsx +++ b/apps/web/app/[locale]/docs/page.tsx @@ -3,6 +3,20 @@ import { Header } from '@/components/Header'; import { Footer } from '@/components/Footer'; import { BookOpen, Terminal, Code } from 'lucide-react'; import Link from 'next/link'; +import { getPageAlternates } from '@/lib/seo'; + + + +export async function generateMetadata({ + params, +}: { + params: Promise<{ locale: string }>; +}) { + const { locale } = await params; + return { + alternates: getPageAlternates(locale, '/docs'), + }; +} export default async function DocsPage({ params, diff --git a/apps/web/app/[locale]/favorites/page.tsx b/apps/web/app/[locale]/favorites/page.tsx index 93f6603..944f778 100644 --- a/apps/web/app/[locale]/favorites/page.tsx +++ b/apps/web/app/[locale]/favorites/page.tsx @@ -6,6 +6,8 @@ import { Footer } from '@/components/Footer'; import { FavoritesList } from '@/components/FavoritesList'; import { FavoritesSignIn } from '@/components/FavoritesSignIn'; import { createDb, userQueries } from '@skillhub/db'; +import { getPageAlternates } from '@/lib/seo'; + // Force dynamic rendering export const dynamic = 'force-dynamic'; @@ -14,6 +16,18 @@ interface FavoritesPageProps { params: Promise<{ locale: string }>; } + +export async function generateMetadata({ + params, +}: { + params: Promise<{ locale: string }>; +}) { + const { locale } = await params; + return { + alternates: getPageAlternates(locale, '/favorites'), + }; +} + export default async function FavoritesPage({ params }: FavoritesPageProps) { const { locale } = await params; setRequestLocale(locale); diff --git a/apps/web/app/[locale]/featured/page.tsx b/apps/web/app/[locale]/featured/page.tsx index 7f48c83..e14ccd7 100644 --- a/apps/web/app/[locale]/featured/page.tsx +++ b/apps/web/app/[locale]/featured/page.tsx @@ -5,6 +5,8 @@ import { createDb, skillQueries } from '@skillhub/db'; import { toPersianNumber } from '@/lib/format-number'; import { Pagination } from '@/components/BrowseFilters'; import { SkillCard } from '@/components/SkillCard'; +import { getPageAlternates } from '@/lib/seo'; + // Force dynamic rendering to fetch fresh data from database export const dynamic = 'force-dynamic'; @@ -37,6 +39,18 @@ async function getFeaturedSkills(page: number, limit: number) { } } + +export async function generateMetadata({ + params, +}: { + params: Promise<{ locale: string }>; +}) { + const { locale } = await params; + return { + alternates: getPageAlternates(locale, '/featured'), + }; +} + export default async function FeaturedPage({ params, searchParams, diff --git a/apps/web/app/[locale]/new/page.tsx b/apps/web/app/[locale]/new/page.tsx index f4e21bc..7c46d43 100644 --- a/apps/web/app/[locale]/new/page.tsx +++ b/apps/web/app/[locale]/new/page.tsx @@ -7,6 +7,8 @@ import { Clock, RefreshCw } from 'lucide-react'; import { createDb, skillQueries } from '@skillhub/db'; import { toPersianNumber } from '@/lib/format-number'; import { Pagination } from '@/components/BrowseFilters'; +import { getPageAlternates } from '@/lib/seo'; + // Force dynamic rendering to fetch fresh data from database export const dynamic = 'force-dynamic'; @@ -75,6 +77,18 @@ async function getTabCounts() { } } + +export async function generateMetadata({ + params, +}: { + params: Promise<{ locale: string }>; +}) { + const { locale } = await params; + return { + alternates: getPageAlternates(locale, '/new'), + }; +} + export default async function NewSkillsPage({ params, searchParams, diff --git a/apps/web/app/[locale]/owner/[username]/page.tsx b/apps/web/app/[locale]/owner/[username]/page.tsx index 52c49d3..bb1b1e3 100644 --- a/apps/web/app/[locale]/owner/[username]/page.tsx +++ b/apps/web/app/[locale]/owner/[username]/page.tsx @@ -7,9 +7,23 @@ import { createDb, skillQueries } from '@skillhub/db'; import { formatCompactNumber, toPersianNumber } from '@/lib/format-number'; import { ExternalLink, Download, Package, Eye, GitFork, ArrowUpDown, FolderGit2 } from 'lucide-react'; import Link from 'next/link'; +import type { Metadata } from 'next'; +import { getPageAlternates } from '@/lib/seo'; export const dynamic = 'force-dynamic'; +export async function generateMetadata({ + params, +}: { + params: Promise<{ locale: string; username: string }>; +}): Promise { + const { locale, username } = await params; + return { + title: `${decodeURIComponent(username)} | SkillHub`, + alternates: getPageAlternates(locale, `/owner/${username}`), + }; +} + const ITEMS_PER_PAGE = 24; type SortOption = 'popularity' | 'downloads' | 'stars'; @@ -183,11 +197,10 @@ export default async function OwnerPage({ params, searchParams }: OwnerPageProps {t('repo.filter')}: {t('repo.all')} ({locale === 'fa' ? toPersianNumber(stats.totalSkills) : stats.totalSkills}) @@ -195,11 +208,10 @@ export default async function OwnerPage({ params, searchParams }: OwnerPageProps {r.repo} ({locale === 'fa' ? toPersianNumber(r.skillCount) : r.skillCount}) @@ -227,11 +239,10 @@ export default async function OwnerPage({ params, searchParams }: OwnerPageProps {opt.label} diff --git a/apps/web/app/[locale]/page.tsx b/apps/web/app/[locale]/page.tsx index 0c12519..f395eb6 100644 --- a/apps/web/app/[locale]/page.tsx +++ b/apps/web/app/[locale]/page.tsx @@ -7,6 +7,8 @@ import { HeroSearch } from '@/components/HeroSearch'; import { SkillCard } from '@/components/SkillCard'; import { createDb, categoryQueries, skillQueries, skills, sql } from '@skillhub/db'; import { formatCompactNumber } from '@/lib/format-number'; +import { getPageAlternates } from '@/lib/seo'; + // Force dynamic rendering to fetch fresh data from database export const dynamic = 'force-dynamic'; @@ -16,8 +18,8 @@ async function getStats() { try { const db = createDb(); - // Browse-ready filter: exclude duplicates and aggregators - const browseReady = sql`${skills.isDuplicate} = false AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`; + // Browse-ready filter: exclude duplicates (matches browseReadyFilter in queries.ts) + const browseReady = sql`${skills.isDuplicate} = false`; // Get total skills count (browse-ready, SKILL.md only) const skillsResult = await db @@ -82,6 +84,18 @@ async function getFeaturedSkills() { } + +export async function generateMetadata({ + params, +}: { + params: Promise<{ locale: string }>; +}) { + const { locale } = await params; + return { + alternates: getPageAlternates(locale, '/'), + }; +} + export default async function HomePage({ params, }: { diff --git a/apps/web/app/[locale]/privacy/page.tsx b/apps/web/app/[locale]/privacy/page.tsx index da400d7..b1ce538 100644 --- a/apps/web/app/[locale]/privacy/page.tsx +++ b/apps/web/app/[locale]/privacy/page.tsx @@ -3,6 +3,8 @@ import Link from 'next/link'; import { Header } from '@/components/Header'; import { Footer } from '@/components/Footer'; import { Shield, Database, Cookie, Users, Lock, Mail, ArrowRight, ArrowLeft } from 'lucide-react'; +import { getPageAlternates } from '@/lib/seo'; + export const dynamic = 'force-dynamic'; @@ -15,6 +17,18 @@ const sectionIcons = { contact: Mail, }; + +export async function generateMetadata({ + params, +}: { + params: Promise<{ locale: string }>; +}) { + const { locale } = await params; + return { + alternates: getPageAlternates(locale, '/privacy'), + }; +} + export default async function PrivacyPage({ params, }: { diff --git a/apps/web/app/[locale]/skill/[...id]/page.tsx b/apps/web/app/[locale]/skill/[...id]/page.tsx index a09f2e9..afefd7f 100644 --- a/apps/web/app/[locale]/skill/[...id]/page.tsx +++ b/apps/web/app/[locale]/skill/[...id]/page.tsx @@ -16,10 +16,32 @@ import { createDb, skillQueries } from '@skillhub/db'; import { FORMAT_LABELS } from 'skillhub-core'; import { formatCompactNumber } from '@/lib/format-number'; import { shouldCountView } from '@/lib/cache'; +import type { Metadata } from 'next'; +import { getPageAlternates } from '@/lib/seo'; // Force dynamic rendering to fetch fresh data from database export const dynamic = 'force-dynamic'; +export async function generateMetadata({ + params, +}: { + params: Promise<{ locale: string; id: string[] }>; +}): Promise { + const { locale, id } = await params; + const skillId = id.join('/'); + + // Optionally fetch skill name for title + const dbSkill = await getSkill(skillId); + const title = dbSkill ? `${dbSkill.name} | SkillHub` : 'SkillHub'; + const description = dbSkill ? dbSkill.description : undefined; + + return { + title, + description, + alternates: getPageAlternates(locale, `/skill/${skillId}`), + }; +} + interface SkillPageProps { params: Promise<{ locale: string; id: string[] }>; } @@ -71,9 +93,9 @@ export default async function SkillPage({ params }: SkillPageProps) { const db = createDb(); shouldCountView(dbSkill.id, clientIp).then((shouldCount) => { if (shouldCount) { - skillQueries.incrementViews(db, dbSkill.id).catch(() => {}); + skillQueries.incrementViews(db, dbSkill.id).catch(() => { }); } - }).catch(() => {}); + }).catch(() => { }); } // Map database response to expected format diff --git a/apps/web/app/[locale]/support/page.tsx b/apps/web/app/[locale]/support/page.tsx index ed3c7b6..a8c73f2 100644 --- a/apps/web/app/[locale]/support/page.tsx +++ b/apps/web/app/[locale]/support/page.tsx @@ -2,6 +2,8 @@ import { getTranslations, setRequestLocale } from 'next-intl/server'; import { Header } from '@/components/Header'; import { Footer } from '@/components/Footer'; import { Mail, Bitcoin, ExternalLink } from 'lucide-react'; +import { getPageAlternates } from '@/lib/seo'; + export const dynamic = 'force-dynamic'; @@ -9,6 +11,18 @@ interface SupportPageProps { params: Promise<{ locale: string }>; } + +export async function generateMetadata({ + params, +}: { + params: Promise<{ locale: string }>; +}) { + const { locale } = await params; + return { + alternates: getPageAlternates(locale, '/support'), + }; +} + export default async function SupportPage({ params }: SupportPageProps) { const { locale } = await params; setRequestLocale(locale); diff --git a/apps/web/app/[locale]/terms/page.tsx b/apps/web/app/[locale]/terms/page.tsx index c8644be..9700781 100644 --- a/apps/web/app/[locale]/terms/page.tsx +++ b/apps/web/app/[locale]/terms/page.tsx @@ -3,6 +3,8 @@ import Link from 'next/link'; import { Header } from '@/components/Header'; import { Footer } from '@/components/Footer'; import { FileText, UserCheck, AlertTriangle, Scale, Trash2, RefreshCw, ArrowRight, ArrowLeft } from 'lucide-react'; +import { getPageAlternates } from '@/lib/seo'; + export const dynamic = 'force-dynamic'; @@ -15,6 +17,18 @@ const sectionIcons = { changes: RefreshCw, }; + +export async function generateMetadata({ + params, +}: { + params: Promise<{ locale: string }>; +}) { + const { locale } = await params; + return { + alternates: getPageAlternates(locale, '/terms'), + }; +} + export default async function TermsPage({ params, }: { diff --git a/apps/web/app/api/stats/route.ts b/apps/web/app/api/stats/route.ts index 0b3c279..559406c 100644 --- a/apps/web/app/api/stats/route.ts +++ b/apps/web/app/api/stats/route.ts @@ -34,8 +34,8 @@ export async function GET(request: NextRequest) { }); } - // Browse-ready filter: exclude duplicates and aggregators - const browseReady = sql`${skills.isDuplicate} = false AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`; + // Browse-ready filter: exclude duplicates (matches browseReadyFilter in queries.ts) + const browseReady = sql`${skills.isDuplicate} = false`; // Browse-ready skill stats (skills count + contributors) const statsResult = await db diff --git a/apps/web/app/sitemap.ts b/apps/web/app/sitemap.ts index 075caa6..4ff85c4 100644 --- a/apps/web/app/sitemap.ts +++ b/apps/web/app/sitemap.ts @@ -1,17 +1,8 @@ import type { MetadataRoute } from 'next'; import { createDb, skillQueries, categoryQueries } from '@skillhub/db'; - -const BASE_URL = - process.env.NEXT_PUBLIC_APP_URL || 'https://skills.palebluedot.live'; +import { getCanonicalPath, primaryDomain as BASE_URL } from '@/lib/seo'; const locales = ['en', 'fa'] as const; -const DEFAULT_LOCALE = 'en'; - -/** Get canonical URL path for a locale (no prefix for default locale, matching localePrefix: 'as-needed') */ -function canonicalPath(locale: string, route: string): string { - if (locale === DEFAULT_LOCALE) return route || '/'; - return `/${locale}${route}`; -} const staticRoutes = [ '', @@ -43,13 +34,13 @@ function makeEntry( } ): MetadataRoute.Sitemap[number] { return { - url: `${BASE_URL}${canonicalPath(locale, route)}`, + url: `${BASE_URL}${getCanonicalPath(locale, route)}`, lastModified: options?.lastModified, changeFrequency: options?.changeFrequency, priority: options?.priority, alternates: { languages: Object.fromEntries( - locales.map((l) => [l, `${BASE_URL}${canonicalPath(l, route)}`]) + locales.map((l) => [l, `${BASE_URL}${getCanonicalPath(l, route)}`]) ), }, }; diff --git a/apps/web/lib/seo.ts b/apps/web/lib/seo.ts new file mode 100644 index 0000000..f8740a5 --- /dev/null +++ b/apps/web/lib/seo.ts @@ -0,0 +1,50 @@ +import { locales } from '@/i18n'; + +export const primaryDomain = + process.env.NEXT_PUBLIC_APP_URL || 'https://skills.palebluedot.live'; + +const DEFAULT_LOCALE = 'en'; + +/** + * Get canonical URL path for a locale + * (no prefix for default locale, matching localePrefix: 'as-needed') + */ +export function getCanonicalPath(locale: string, route: string): string { + // Normalize route to always start with a slash and not end with a slash + let formattedRoute = route; + if (!formattedRoute.startsWith('/')) { + formattedRoute = `/${formattedRoute}`; + } + if (formattedRoute.length > 1 && formattedRoute.endsWith('/')) { + formattedRoute = formattedRoute.slice(0, -1); + } + + if (locale === DEFAULT_LOCALE) { + if (formattedRoute === '/') return ''; + return formattedRoute; + } + + if (formattedRoute === '/') return `/${locale}`; + return `/${locale}${formattedRoute}`; +} + +/** + * Generate Next.js alternates metadata for a specific route + * @param locale Current active locale + * @param route The route path (e.g. '/about', '/categories') + */ +export function getPageAlternates(locale: string, route: string) { + // Ensure we fall back to / if it's empty + const canonicalPath = getCanonicalPath(locale, route) || '/'; + + const languages: Record = {}; + + for (const l of locales) { + languages[l] = `${primaryDomain}${getCanonicalPath(l, route) || '/'}`; + } + + return { + canonical: `${primaryDomain}${canonicalPath}`, + languages, + }; +} diff --git a/packages/db/src/queries.ts b/packages/db/src/queries.ts index 38027a4..2ea9eea 100644 --- a/packages/db/src/queries.ts +++ b/packages/db/src/queries.ts @@ -31,11 +31,11 @@ function getRawClientLocal() { type DB = PostgresJsDatabase; /** - * Curation filter: excludes duplicates and aggregator-only skills. + * Curation filter: excludes duplicates. * Skills with skill_type=NULL (newly crawled, not yet curated) are included * so they don't disappear between curate.mjs runs. */ -const browseReadyFilter = sql`(${skills.isDuplicate} = false) AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`; +const browseReadyFilter = sql`(${skills.isDuplicate} = false)`; /** * Skill queries @@ -556,16 +556,37 @@ export const skillQueries = { skill: typeof skills.$inferInsert ): Promise => { // Check if commitSha changed (for cache invalidation) + // and if content_hash changed (for re-review mechanism) let shouldClearCache = false; - if (skill.id && skill.commitSha) { + let shouldTriggerReReview = false; + if (skill.id) { const existing = await db - .select({ commitSha: skills.commitSha, cachedFiles: skills.cachedFiles }) + .select({ + commitSha: skills.commitSha, + cachedFiles: skills.cachedFiles, + contentHash: skills.contentHash, + reviewStatus: skills.reviewStatus, + }) .from(skills) .where(eq(skills.id, skill.id)) .limit(1); - if (existing[0] && existing[0].cachedFiles && existing[0].commitSha !== skill.commitSha) { - shouldClearCache = true; + if (existing[0]) { + // Cache invalidation: clear cachedFiles when commitSha changes + if (skill.commitSha && existing[0].cachedFiles && existing[0].commitSha !== skill.commitSha) { + shouldClearCache = true; + } + + // Re-review mechanism (Phase 6.4): when content_hash changes and skill + // was previously reviewed, mark it for re-review + if ( + skill.contentHash && + existing[0].contentHash && + existing[0].contentHash !== skill.contentHash && + (existing[0].reviewStatus === 'verified' || existing[0].reviewStatus === 'ai-reviewed') + ) { + shouldTriggerReReview = true; + } } } @@ -605,6 +626,9 @@ export const skillQueries = { // which only updates it when content-related columns actually change // Clear cachedFiles if commitSha changed ...(shouldClearCache ? { cachedFiles: null } : {}), + // Re-review mechanism (Phase 6.4): mark previously reviewed skills for re-review + // when their content changes. Only affects 'verified' and 'ai-reviewed' skills. + ...(shouldTriggerReReview ? { reviewStatus: 'needs-re-review' } : {}), }, }) .returning(); @@ -1172,7 +1196,7 @@ export async function runPostCrawlCuration(db: DB): Promise<{ classified: number `); results.classified = (classResult as unknown as { rowCount: number }).rowCount ?? 0; - // Step 4: Mark duplicates by content_hash (keep canonical = most stars) + // Step 4: Mark duplicates by content_hash (keep canonical = most stars, prioritizing standalone over aggregator) const dupResult = await db.execute(sql` UPDATE skills s SET is_duplicate = true WHERE s.is_blocked = false @@ -1183,8 +1207,17 @@ export async function runPostCrawlCuration(db: DB): Promise<{ classified: number WHERE s2.content_hash = s.content_hash AND s2.is_blocked = false AND s2.id != s.id - AND (s2.github_stars > s.github_stars - OR (s2.github_stars = s.github_stars AND s2.indexed_at < s.indexed_at)) + AND ( + (s2.skill_type != 'aggregator' AND s.skill_type = 'aggregator') + OR ( + (s2.skill_type = 'aggregator') = (s.skill_type = 'aggregator') + AND ( + s2.github_stars > s.github_stars + OR (s2.github_stars = s.github_stars AND s2.github_forks > s.github_forks) + OR (s2.github_stars = s.github_stars AND s2.github_forks = s.github_forks AND s2.created_at < s.created_at) + ) + ) + ) ) `); results.duplicates = (dupResult as unknown as { rowCount: number }).rowCount ?? 0; @@ -1198,7 +1231,6 @@ export async function runPostCrawlCuration(db: DB): Promise<{ classified: number JOIN skills s ON sc.skill_id = s.id WHERE s.is_blocked = false AND s.is_duplicate = false - AND (s.skill_type IS NULL OR s.skill_type != 'aggregator') GROUP BY sc.category_id ) sub WHERE c.id = sub.category_id diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 396237d..b42a98e 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -79,6 +79,7 @@ export const skills = pgTable( isDuplicate: boolean('is_duplicate').default(false), canonicalSkillId: text('canonical_skill_id'), // points to the "original" if this is a duplicate repoSkillCount: integer('repo_skill_count'), // cached count of skills in same repo + reviewStatus: text('review_status').default('unreviewed'), // unreviewed → auto-scored → ai-reviewed → verified (or needs-re-review) // Content (cached) contentHash: text('content_hash'), diff --git a/scripts/curation/batch-score.mjs b/scripts/curation/batch-score.mjs new file mode 100644 index 0000000..1e6c402 --- /dev/null +++ b/scripts/curation/batch-score.mjs @@ -0,0 +1,459 @@ +#!/usr/bin/env node +/** + * Phase 5: Batch Quality Score + * + * Calculates quality scores for browse-ready skills using the same logic + * as SkillAnalyzer.calculateQuality() from services/indexer/src/analyzer.ts. + * + * Updates: quality_score, quality_details, review_status → 'auto-scored' + * + * Usage: + * DATABASE_URL=postgres://... node scripts/curation/batch-score.mjs + * + * Options: + * --dry-run Show what would change without writing + * --batch-size=N Process N skills per batch (default 100) + */ + +import { createRequire } from 'module'; +import { resolve, dirname } from 'path'; +import { fileURLToPath, pathToFileURL } from 'url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +// ─── Find pg module ─── +let pg; +const tryPaths = [ + resolve(__dirname, '../..', 'package.json'), + '/tmp/package.json', + process.env.APPDATA ? resolve(process.env.APPDATA, '..', 'Local', 'Temp', 'package.json') : null, +].filter(Boolean); +for (const p of tryPaths) { + try { pg = createRequire(p)('pg'); break; } catch {} +} +if (!pg) { console.error('pg not found. Run: npm install pg'); process.exit(1); } + +// ─── Find skillhub-core (ESM-only — use direct path import) ─── +let parseSkillMd, parseGenericInstructionFile, validateSkill, scanSecurity; +const corePaths = [ + resolve(__dirname, '../../packages/core/dist/index.js'), + resolve(__dirname, '../../node_modules/skillhub-core/dist/index.js'), + resolve(__dirname, '../../services/indexer/node_modules/skillhub-core/dist/index.js'), +]; +for (const p of corePaths) { + try { + const core = await import(pathToFileURL(p).href); + parseSkillMd = core.parseSkillMd; + parseGenericInstructionFile = core.parseGenericInstructionFile; + validateSkill = core.validateSkill; + scanSecurity = core.scanSecurity; + break; + } catch {} +} +if (!parseSkillMd) { console.error('skillhub-core not found. Run: pnpm build'); process.exit(1); } + +// ─── Config ─── +const DATABASE_URL = process.env.DATABASE_URL || 'postgresql://postgres:postgres@localhost:5432/skillhub'; +const DRY_RUN = process.argv.includes('--dry-run'); +const batchArg = process.argv.find(a => a.startsWith('--batch-size=')); +const BATCH_SIZE = batchArg ? parseInt(batchArg.split('=')[1]) : 100; + +// ─── Helpers ─── +const n = v => Number(v ?? 0).toLocaleString('en-US'); + +function progress(current, total) { + const pct = ((current / total) * 100).toFixed(1); + process.stdout.write(`\r Processing ${n(current)} / ${n(total)} (${pct}%)`); +} + +// ─── Database ─── +let client; + +async function connect() { + client = new pg.Client({ + connectionString: DATABASE_URL, + ssl: false, + connectionTimeoutMillis: 15000, + query_timeout: 600000, + keepAlive: true, + }); + await client.connect(); + console.log('Connected to database'); +} + +async function query(sql, params = []) { + return client.query(sql, params); +} + +// ─── Quality scoring (mirrors SkillAnalyzer.calculateQuality) ─── + +function scoreDocumentation(skill, scripts, references) { + let score = 0; + + // Has description (required) + if (skill.metadata.description && skill.metadata.description.length > 20) { + score += 20; + } + + // Content length and structure + const contentLength = skill.content.length; + if (contentLength > 500) score += 15; + else if (contentLength > 200) score += 10; + else if (contentLength > 50) score += 5; + + // Has headers (good structure) + const headerCount = (skill.content.match(/^#+\s/gm) || []).length; + if (headerCount >= 3) score += 15; + else if (headerCount >= 1) score += 10; + + // Has code examples + if (skill.content.includes('```')) { + score += 15; + } + + // Has version + if (skill.metadata.version) { + score += 10; + } + + // Has license + if (skill.metadata.license) { + score += 5; + } + + // Has compatibility info + if (skill.metadata.compatibility?.platforms?.length) { + score += 10; + } + + // Has scripts + if (scripts.length > 0) { + score += 5; + } + + // Has references + if (references.length > 0) { + score += 5; + } + + return Math.min(100, score); +} + +function scoreMaintenance(repoMeta) { + let score = 0; + + // Check last update time + const lastUpdate = new Date(repoMeta.updatedAt); + const daysSinceUpdate = (Date.now() - lastUpdate.getTime()) / (1000 * 60 * 60 * 24); + + if (daysSinceUpdate < 30) score += 40; + else if (daysSinceUpdate < 90) score += 30; + else if (daysSinceUpdate < 180) score += 20; + else if (daysSinceUpdate < 365) score += 10; + + // Has license + if (repoMeta.license) { + score += 20; + } + + // Has description + if (repoMeta.description) { + score += 10; + } + + // Has topics (always [] since not in DB — will miss ~10 points) + if (repoMeta.topics.length > 0) { + score += 10; + } + + // Activity level (forks indicate usage) + if (repoMeta.forks >= 10) score += 20; + else if (repoMeta.forks >= 5) score += 15; + else if (repoMeta.forks >= 1) score += 10; + + return Math.min(100, score); +} + +function scorePopularity(repoMeta) { + const stars = repoMeta.stars; + const forks = repoMeta.forks; + + let score = 0; + + if (stars >= 1000) score += 50; + else if (stars >= 100) score += 40; + else if (stars >= 50) score += 30; + else if (stars >= 10) score += 20; + else if (stars >= 5) score += 10; + else if (stars >= 1) score += 5; + + if (forks >= 50) score += 30; + else if (forks >= 10) score += 20; + else if (forks >= 5) score += 15; + else if (forks >= 1) score += 10; + + // Bonus for relevant topics (always [] from DB — will miss ~20 points) + const relevantTopics = ['ai', 'agent', 'skill', 'claude', 'copilot', 'codex', 'llm']; + const hasRelevantTopic = repoMeta.topics.some(t => + relevantTopics.some(rt => t.toLowerCase().includes(rt)) + ); + if (hasRelevantTopic) { + score += 20; + } + + return Math.min(100, score); +} + +function calculateQuality(skill, repoMeta, securityScore, validation, scripts, references) { + const factors = []; + + // Documentation quality (30% weight) + const docScore = scoreDocumentation(skill, scripts, references); + factors.push({ name: 'documentation', score: docScore, weight: 0.3 }); + + // Maintenance signals (25% weight) + const maintScore = scoreMaintenance(repoMeta); + factors.push({ name: 'maintenance', score: maintScore, weight: 0.25 }); + + // Popularity (20% weight) + const popScore = scorePopularity(repoMeta); + factors.push({ name: 'popularity', score: popScore, weight: 0.2 }); + + // Security (15% weight) + factors.push({ name: 'security', score: securityScore, weight: 0.15 }); + + // Validation (10% weight) + const valScore = validation.isValid ? 100 : Math.max(0, 100 - validation.errors.length * 20); + factors.push({ name: 'validation', score: valScore, weight: 0.1 }); + + // Calculate weighted overall score + const overall = Math.round( + factors.reduce((sum, f) => sum + f.score * f.weight, 0) + ); + + return { + overall, + documentation: docScore, + maintenance: maintScore, + popularity: popScore, + factors, + }; +} + +// ─── Extract scripts/references from cached_files ─── +function extractScripts(cachedFiles) { + if (!cachedFiles || !cachedFiles.items) return []; + return cachedFiles.items + .filter(item => + !item.isBinary && + item.name !== 'SKILL.md' && + (item.name.endsWith('.sh') || + item.name.endsWith('.py') || + item.name.endsWith('.js') || + item.name.endsWith('.ts') || + item.name.endsWith('.ps1') || + item.name.endsWith('.bat') || + item.name.endsWith('.rb')) + ) + .map(item => ({ name: item.name, content: item.content })); +} + +function extractReferences(cachedFiles) { + if (!cachedFiles || !cachedFiles.items) return []; + return cachedFiles.items + .filter(item => + !item.isBinary && + item.name !== 'SKILL.md' && + (item.name.endsWith('.md') || item.name.endsWith('.txt')) + ) + .map(item => ({ name: item.name, content: item.content })); +} + +// ─── Main ─── +async function main() { + const t0 = Date.now(); + await connect(); + + console.log(`\n${'='.repeat(70)}`); + console.log(' BATCH QUALITY SCORE (Phase 5)'); + console.log(`${'='.repeat(70)}`); + if (DRY_RUN) console.log('\n *** DRY RUN MODE — no changes will be made ***\n'); + + // Count skills to score + const countResult = await query(` + SELECT COUNT(*)::int AS total + FROM skills + WHERE is_blocked = false + AND is_duplicate = false + AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection')) + AND quality_score IS NULL + AND raw_content IS NOT NULL + `); + const total = countResult.rows[0].total; + console.log(` Skills to score: ${n(total)}`); + console.log(` Batch size: ${BATCH_SIZE}`); + + if (total === 0) { + console.log(' Nothing to do — all browse-ready skills already scored'); + await client.end().catch(() => {}); + return; + } + + if (DRY_RUN) { + const sample = await query(` + SELECT id, LEFT(name, 50) AS name, github_stars + FROM skills + WHERE is_blocked = false + AND is_duplicate = false + AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection')) + AND quality_score IS NULL + AND raw_content IS NOT NULL + ORDER BY github_stars DESC NULLS LAST + LIMIT 5 + `); + console.log('\n Sample skills that would be scored:'); + for (const r of sample.rows) { + console.log(` [${n(r.github_stars)}★] ${r.id} — ${r.name}`); + } + console.log(`\n [DRY RUN] Would score ${n(total)} skills`); + await client.end().catch(() => {}); + return; + } + + // Process in batches + let processed = 0; + let errorCount = 0; + let totalScore = 0; + const scoreBuckets = { excellent: 0, good: 0, fair: 0, poor: 0 }; + + while (processed < total) { + const batch = await query(` + SELECT id, raw_content, cached_files, source_format, + github_stars, github_forks, license, description, + updated_at, version, compatibility, security_score + FROM skills + WHERE is_blocked = false + AND is_duplicate = false + AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection')) + AND quality_score IS NULL + AND raw_content IS NOT NULL + ORDER BY github_stars DESC NULLS LAST + LIMIT $1 + `, [BATCH_SIZE]); + + if (batch.rows.length === 0) break; + + for (const row of batch.rows) { + try { + // Parse the skill content + const sourceFormat = row.source_format || 'skill.md'; + let skill; + if (sourceFormat === 'skill.md') { + skill = parseSkillMd(row.raw_content); + } else { + skill = parseGenericInstructionFile(row.raw_content, sourceFormat, { + name: row.description?.split(/\s+/).slice(0, 3).join('-') || 'skill', + description: row.description, + owner: '', + }); + } + + // Validate + const validation = sourceFormat === 'skill.md' + ? validateSkill(skill) + : { isValid: true, errors: [], warnings: [] }; + + // Construct pseudo-repoMeta from DB fields + const repoMeta = { + stars: row.github_stars || 0, + forks: row.github_forks || 0, + license: row.license || null, + description: row.description || null, + updatedAt: row.updated_at ? row.updated_at.toISOString() : new Date().toISOString(), + defaultBranch: 'main', + topics: [], // NOT in DB — popularity factor will miss ~20 points + }; + + // Use already-computed security score (from Phase 4), default to 100 if not scanned + const securityScore = row.security_score ?? 100; + + // Extract scripts/references from cached_files + const scripts = extractScripts(row.cached_files); + const references = extractReferences(row.cached_files); + + // Calculate quality + const quality = calculateQuality(skill, repoMeta, securityScore, validation, scripts, references); + + // Build quality_details for JSONB storage + const qualityDetails = { + documentation: quality.documentation, + maintenance: quality.maintenance, + popularity: quality.popularity, + factors: quality.factors, + }; + + await query(` + UPDATE skills + SET quality_score = $1, + quality_details = $2, + review_status = CASE + WHEN review_status IS NULL OR review_status = 'unreviewed' + THEN 'auto-scored' + ELSE review_status + END + WHERE id = $3 + `, [quality.overall, JSON.stringify(qualityDetails), row.id]); + + totalScore += quality.overall; + if (quality.overall >= 70) scoreBuckets.excellent++; + else if (quality.overall >= 50) scoreBuckets.good++; + else if (quality.overall >= 30) scoreBuckets.fair++; + else scoreBuckets.poor++; + } catch (err) { + errorCount++; + if (errorCount <= 5) { + console.error(`\n Error scoring ${row.id}: ${err.message}`); + } + } + + processed++; + if (processed % 100 === 0 || processed === total) { + progress(processed, total); + } + } + } + + console.log('\n'); // Clear progress line + + // Summary + const scored = processed - errorCount; + const avgScore = scored > 0 ? (totalScore / scored).toFixed(1) : 0; + console.log(` + ┌─────────────────────────────────────────────────────┐ + │ QUALITY SCORE SUMMARY │ + ├─────────────────────────────────────────────────────┤ + │ Total scored: ${n(scored).padStart(8)} │ + │ Errors (skipped): ${n(errorCount).padStart(8)} │ + ├─────────────────────────────────────────────────────┤ + │ Excellent (70+): ${n(scoreBuckets.excellent).padStart(8)} │ + │ Good (50-69): ${n(scoreBuckets.good).padStart(8)} │ + │ Fair (30-49): ${n(scoreBuckets.fair).padStart(8)} │ + │ Poor (<30): ${n(scoreBuckets.poor).padStart(8)} │ + ├─────────────────────────────────────────────────────┤ + │ Avg quality score: ${avgScore.toString().padStart(8)} │ + └─────────────────────────────────────────────────────┘ + `); + + console.log(' Note: repoMeta.topics not in DB — popularity score misses ~20 points'); + + const dur = ((Date.now() - t0) / 1000).toFixed(1); + console.log(`\nCompleted in ${dur}s`); + + await client.end().catch(() => {}); +} + +main().catch(async e => { + console.error('FAILED:', e.message || e); + await client?.end().catch(() => {}); + process.exit(1); +}); diff --git a/scripts/curation/batch-security.mjs b/scripts/curation/batch-security.mjs new file mode 100644 index 0000000..76e487d --- /dev/null +++ b/scripts/curation/batch-security.mjs @@ -0,0 +1,240 @@ +#!/usr/bin/env node +/** + * Phase 4: Batch Security Scan + * + * Scans browse-ready skills for security issues using scanSecurity() from skillhub-core. + * Updates: security_score, security_status, last_scanned, review_status → 'auto-scored' + * + * Usage: + * DATABASE_URL=postgres://... node scripts/curation/batch-security.mjs + * + * Options: + * --dry-run Show what would change without writing + * --batch-size=N Process N skills per batch (default 100) + */ + +import { createRequire } from 'module'; +import { resolve, dirname } from 'path'; +import { fileURLToPath, pathToFileURL } from 'url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +// ─── Find pg module ─── +let pg; +const tryPaths = [ + resolve(__dirname, '../..', 'package.json'), + '/tmp/package.json', + process.env.APPDATA ? resolve(process.env.APPDATA, '..', 'Local', 'Temp', 'package.json') : null, +].filter(Boolean); +for (const p of tryPaths) { + try { pg = createRequire(p)('pg'); break; } catch {} +} +if (!pg) { console.error('pg not found. Run: npm install pg'); process.exit(1); } + +// ─── Find skillhub-core (ESM-only — use direct path import) ─── +let scanSecurity; +const corePaths = [ + resolve(__dirname, '../../packages/core/dist/index.js'), + resolve(__dirname, '../../node_modules/skillhub-core/dist/index.js'), + resolve(__dirname, '../../services/indexer/node_modules/skillhub-core/dist/index.js'), +]; +for (const p of corePaths) { + try { + const core = await import(pathToFileURL(p).href); + scanSecurity = core.scanSecurity; + break; + } catch {} +} +if (!scanSecurity) { console.error('skillhub-core not found. Run: pnpm build'); process.exit(1); } + +// ─── Config ─── +const DATABASE_URL = process.env.DATABASE_URL || 'postgresql://postgres:postgres@localhost:5432/skillhub'; +const DRY_RUN = process.argv.includes('--dry-run'); +const batchArg = process.argv.find(a => a.startsWith('--batch-size=')); +const BATCH_SIZE = batchArg ? parseInt(batchArg.split('=')[1]) : 100; + +// ─── Helpers ─── +const n = v => Number(v ?? 0).toLocaleString('en-US'); + +function progress(current, total) { + const pct = ((current / total) * 100).toFixed(1); + process.stdout.write(`\r Processing ${n(current)} / ${n(total)} (${pct}%)`); +} + +// ─── Database ─── +let client; + +async function connect() { + client = new pg.Client({ + connectionString: DATABASE_URL, + ssl: false, + connectionTimeoutMillis: 15000, + query_timeout: 600000, + keepAlive: true, + }); + await client.connect(); + console.log('Connected to database'); +} + +async function query(sql, params = []) { + return client.query(sql, params); +} + +// ─── Extract scripts from cached_files ─── +function extractScripts(cachedFiles) { + if (!cachedFiles || !cachedFiles.items) return []; + return cachedFiles.items + .filter(item => + !item.isBinary && + item.name !== 'SKILL.md' && + (item.name.endsWith('.sh') || + item.name.endsWith('.py') || + item.name.endsWith('.js') || + item.name.endsWith('.ts') || + item.name.endsWith('.ps1') || + item.name.endsWith('.bat') || + item.name.endsWith('.rb')) + ) + .map(item => ({ name: item.name, content: item.content })); +} + +// ─── Main ─── +async function main() { + const t0 = Date.now(); + await connect(); + + console.log(`\n${'='.repeat(70)}`); + console.log(' BATCH SECURITY SCAN (Phase 4)'); + console.log(`${'='.repeat(70)}`); + if (DRY_RUN) console.log('\n *** DRY RUN MODE — no changes will be made ***\n'); + + // Count skills to scan + const countResult = await query(` + SELECT COUNT(*)::int AS total + FROM skills + WHERE is_blocked = false + AND is_duplicate = false + AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection')) + AND security_status IS NULL + AND raw_content IS NOT NULL + `); + const total = countResult.rows[0].total; + console.log(` Skills to scan: ${n(total)}`); + console.log(` Batch size: ${BATCH_SIZE}`); + + if (total === 0) { + console.log(' Nothing to do — all browse-ready skills already scanned'); + await client.end().catch(() => {}); + return; + } + + if (DRY_RUN) { + // Show sample + const sample = await query(` + SELECT id, LEFT(name, 50) AS name + FROM skills + WHERE is_blocked = false + AND is_duplicate = false + AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection')) + AND security_status IS NULL + AND raw_content IS NOT NULL + LIMIT 5 + `); + console.log('\n Sample skills that would be scanned:'); + for (const r of sample.rows) { + console.log(` ${r.id} — ${r.name}`); + } + console.log(`\n [DRY RUN] Would scan ${n(total)} skills`); + await client.end().catch(() => {}); + return; + } + + // Process in batches + let processed = 0; + let passCount = 0; + let warnCount = 0; + let failCount = 0; + let errorCount = 0; + let totalScore = 0; + + while (processed < total) { + const batch = await query(` + SELECT id, raw_content, cached_files + FROM skills + WHERE is_blocked = false + AND is_duplicate = false + AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection')) + AND security_status IS NULL + AND raw_content IS NOT NULL + ORDER BY github_stars DESC NULLS LAST + LIMIT $1 + `, [BATCH_SIZE]); + + if (batch.rows.length === 0) break; + + for (const row of batch.rows) { + try { + const scripts = extractScripts(row.cached_files); + const report = scanSecurity({ content: row.raw_content, scripts }); + + await query(` + UPDATE skills + SET security_score = $1, + security_status = $2, + last_scanned = NOW(), + review_status = CASE + WHEN review_status IS NULL OR review_status = 'unreviewed' + THEN 'auto-scored' + ELSE review_status + END + WHERE id = $3 + `, [report.score, report.status, row.id]); + + totalScore += report.score; + if (report.status === 'pass') passCount++; + else if (report.status === 'warning') warnCount++; + else failCount++; + } catch (err) { + errorCount++; + if (errorCount <= 5) { + console.error(`\n Error scanning ${row.id}: ${err.message}`); + } + } + + processed++; + if (processed % 100 === 0 || processed === total) { + progress(processed, total); + } + } + } + + console.log('\n'); // Clear progress line + + // Summary + const avgScore = processed > 0 ? (totalScore / (processed - errorCount)).toFixed(1) : 0; + console.log(` + ┌─────────────────────────────────────────────────────┐ + │ SECURITY SCAN SUMMARY │ + ├─────────────────────────────────────────────────────┤ + │ Total scanned: ${n(processed).padStart(8)} │ + │ Errors (skipped): ${n(errorCount).padStart(8)} │ + ├─────────────────────────────────────────────────────┤ + │ PASS: ${n(passCount).padStart(8)} │ + │ WARNING: ${n(warnCount).padStart(8)} │ + │ FAIL: ${n(failCount).padStart(8)} │ + ├─────────────────────────────────────────────────────┤ + │ Avg security score:${avgScore.toString().padStart(8)} │ + └─────────────────────────────────────────────────────┘ + `); + + const dur = ((Date.now() - t0) / 1000).toFixed(1); + console.log(`Completed in ${dur}s`); + + await client.end().catch(() => {}); +} + +main().catch(async e => { + console.error('FAILED:', e.message || e); + await client?.end().catch(() => {}); + process.exit(1); +}); diff --git a/scripts/init-db.sql b/scripts/init-db.sql index c6c233d..6ac8821 100644 --- a/scripts/init-db.sql +++ b/scripts/init-db.sql @@ -51,6 +51,9 @@ CREATE TABLE IF NOT EXISTS skills ( is_blocked BOOLEAN DEFAULT FALSE, -- Blocked from re-indexing (owner requested removal) last_scanned TIMESTAMP WITH TIME ZONE, + -- Review pipeline + review_status TEXT DEFAULT 'unreviewed', -- 'unreviewed', 'auto-scored', 'ai-reviewed', 'verified', 'needs-re-review' + -- Content content_hash TEXT, raw_content TEXT, diff --git a/services/indexer/Dockerfile b/services/indexer/Dockerfile index 617e469..51ea937 100644 --- a/services/indexer/Dockerfile +++ b/services/indexer/Dockerfile @@ -44,6 +44,11 @@ RUN adduser --system --uid 1001 indexer # Copy only the bundled dist (no node_modules needed - everything is bundled) COPY --from=builder --chown=indexer:nodejs /app/services/indexer/dist ./dist +# Curation scripts + skillhub-core (for batch-security, batch-score, curate) +COPY --from=builder --chown=indexer:nodejs /app/scripts/curation ./scripts/curation +COPY --from=builder --chown=indexer:nodejs /app/packages/core/dist ./packages/core/dist +RUN npm install --no-save pg gray-matter yaml + USER indexer CMD ["node", "dist/worker.js"]