sync: SEO canonical URLs, curation improvements, batch scripts, stats fix
- Dynamic canonical URLs and alternates for all pages (SEO fix) - Shared seo.ts utility for canonical path generation - Improved duplicate detection: standalone > aggregator priority, forks/created_at tie-breakers - browseReadyFilter now includes unique aggregator skills - Stats aligned with browseReadyFilter (was showing 16k instead of 62k+) - Re-review mechanism: content_hash changes trigger needs-re-review - review_status field added to skills schema - Batch security scan and quality score scripts - Indexer Dockerfile updated for curation deps - Removed roadmap from README Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
12
README.md
12
README.md
@@ -184,18 +184,6 @@ Always review skill source code before installing.
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Roadmap
|
|
||||||
|
|
||||||
- [x] Web marketplace with growing skill catalog
|
|
||||||
- [x] CLI tool
|
|
||||||
- [x] Multi-platform support
|
|
||||||
- [x] Security scanning
|
|
||||||
- [ ] Claude Code MCP plugin
|
|
||||||
- [ ] AI-powered recommendations
|
|
||||||
- [ ] Enterprise features
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
MIT - See [LICENSE](./LICENSE) for details.
|
MIT - See [LICENSE](./LICENSE) for details.
|
||||||
|
|||||||
@@ -2,6 +2,20 @@ import { getTranslations, setRequestLocale } from 'next-intl/server';
|
|||||||
import { Header } from '@/components/Header';
|
import { Header } from '@/components/Header';
|
||||||
import { Footer } from '@/components/Footer';
|
import { Footer } from '@/components/Footer';
|
||||||
import { Code2, Globe, Shield } from 'lucide-react';
|
import { Code2, Globe, Shield } from 'lucide-react';
|
||||||
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
export async function generateMetadata({
|
||||||
|
params,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string }>;
|
||||||
|
}) {
|
||||||
|
const { locale } = await params;
|
||||||
|
return {
|
||||||
|
alternates: getPageAlternates(locale, '/about'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export default async function AboutPage({
|
export default async function AboutPage({
|
||||||
params,
|
params,
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ import Link from 'next/link';
|
|||||||
import { Header } from '@/components/Header';
|
import { Header } from '@/components/Header';
|
||||||
import { Footer } from '@/components/Footer';
|
import { Footer } from '@/components/Footer';
|
||||||
import { Github, Heart, Users, Code, GitFork, ExternalLink, Database, Clock } from 'lucide-react';
|
import { Github, Heart, Users, Code, GitFork, ExternalLink, Database, Clock } from 'lucide-react';
|
||||||
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
|
|
||||||
|
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
@@ -65,6 +67,18 @@ function formatLicenseName(license: string, locale: string): string {
|
|||||||
return names ? names[locale as 'en' | 'fa'] || names.en : license;
|
return names ? names[locale as 'en' | 'fa'] || names.en : license;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
export async function generateMetadata({
|
||||||
|
params,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string }>;
|
||||||
|
}) {
|
||||||
|
const { locale } = await params;
|
||||||
|
return {
|
||||||
|
alternates: getPageAlternates(locale, '/attribution'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export default async function AttributionPage({
|
export default async function AttributionPage({
|
||||||
params,
|
params,
|
||||||
}: {
|
}: {
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ import { createDb, skillQueries, categoryQueries } from '@skillhub/db';
|
|||||||
import { BrowseFilters, SearchBar, Pagination, ActiveFilters, EmptyState } from '@/components/BrowseFilters';
|
import { BrowseFilters, SearchBar, Pagination, ActiveFilters, EmptyState } from '@/components/BrowseFilters';
|
||||||
import { SkillCard } from '@/components/SkillCard';
|
import { SkillCard } from '@/components/SkillCard';
|
||||||
import { toPersianNumber } from '@/lib/format-number';
|
import { toPersianNumber } from '@/lib/format-number';
|
||||||
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
|
|
||||||
|
|
||||||
// Force dynamic rendering to fetch fresh data from database
|
// Force dynamic rendering to fetch fresh data from database
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
@@ -79,6 +81,18 @@ async function getSkills(params: {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
export async function generateMetadata({
|
||||||
|
params,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string }>;
|
||||||
|
}) {
|
||||||
|
const { locale } = await params;
|
||||||
|
return {
|
||||||
|
alternates: getPageAlternates(locale, '/browse'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export default async function BrowsePage({ params, searchParams }: BrowsePageProps) {
|
export default async function BrowsePage({ params, searchParams }: BrowsePageProps) {
|
||||||
const { locale } = await params;
|
const { locale } = await params;
|
||||||
setRequestLocale(locale);
|
setRequestLocale(locale);
|
||||||
|
|||||||
@@ -31,6 +31,8 @@ import {
|
|||||||
} from 'lucide-react';
|
} from 'lucide-react';
|
||||||
import { createDb, categoryQueries } from '@skillhub/db';
|
import { createDb, categoryQueries } from '@skillhub/db';
|
||||||
import { formatNumber } from '@/lib/format-number';
|
import { formatNumber } from '@/lib/format-number';
|
||||||
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
|
|
||||||
|
|
||||||
// Force dynamic rendering to fetch fresh data from database
|
// Force dynamic rendering to fetch fresh data from database
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
@@ -96,6 +98,18 @@ async function getHierarchicalCategories() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
export async function generateMetadata({
|
||||||
|
params,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string }>;
|
||||||
|
}) {
|
||||||
|
const { locale } = await params;
|
||||||
|
return {
|
||||||
|
alternates: getPageAlternates(locale, '/categories'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export default async function CategoriesPage({
|
export default async function CategoriesPage({
|
||||||
params,
|
params,
|
||||||
}: {
|
}: {
|
||||||
|
|||||||
@@ -2,9 +2,23 @@ import { getTranslations, setRequestLocale } from 'next-intl/server';
|
|||||||
import { Header } from '@/components/Header';
|
import { Header } from '@/components/Header';
|
||||||
import { Footer } from '@/components/Footer';
|
import { Footer } from '@/components/Footer';
|
||||||
import { ClaimForm } from '@/components/ClaimForm';
|
import { ClaimForm } from '@/components/ClaimForm';
|
||||||
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
|
|
||||||
|
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
|
|
||||||
|
export async function generateMetadata({
|
||||||
|
params,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string }>;
|
||||||
|
}) {
|
||||||
|
const { locale } = await params;
|
||||||
|
return {
|
||||||
|
alternates: getPageAlternates(locale, '/claim'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export default async function ClaimPage({
|
export default async function ClaimPage({
|
||||||
params,
|
params,
|
||||||
}: {
|
}: {
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { getTranslations, setRequestLocale } from 'next-intl/server';
|
import { getTranslations, setRequestLocale } from 'next-intl/server';
|
||||||
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
import Link from 'next/link';
|
import Link from 'next/link';
|
||||||
import type { Metadata } from 'next';
|
import type { Metadata } from 'next';
|
||||||
import {
|
import {
|
||||||
@@ -31,6 +32,7 @@ export async function generateMetadata({
|
|||||||
return {
|
return {
|
||||||
title: t('metadata.title'),
|
title: t('metadata.title'),
|
||||||
description: t('metadata.description'),
|
description: t('metadata.description'),
|
||||||
|
alternates: getPageAlternates(locale, '/claude-plugin'),
|
||||||
openGraph: {
|
openGraph: {
|
||||||
title: t('metadata.title'),
|
title: t('metadata.title'),
|
||||||
description: t('metadata.description'),
|
description: t('metadata.description'),
|
||||||
|
|||||||
@@ -1,4 +1,18 @@
|
|||||||
import { redirect } from 'next/navigation';
|
import { redirect } from 'next/navigation';
|
||||||
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
export async function generateMetadata({
|
||||||
|
params,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string }>;
|
||||||
|
}) {
|
||||||
|
const { locale } = await params;
|
||||||
|
return {
|
||||||
|
alternates: getPageAlternates(locale, '/contact'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export default async function ContactPage({
|
export default async function ContactPage({
|
||||||
params,
|
params,
|
||||||
|
|||||||
@@ -5,6 +5,20 @@ import { ApiEndpointSection } from '@/components/ApiEndpointSection';
|
|||||||
import type { EndpointDef } from '@/components/ApiEndpointSection';
|
import type { EndpointDef } from '@/components/ApiEndpointSection';
|
||||||
import Link from 'next/link';
|
import Link from 'next/link';
|
||||||
import { ArrowLeft, ArrowRight, Search, FileCode, Users, Compass, Mail } from 'lucide-react';
|
import { ArrowLeft, ArrowRight, Search, FileCode, Users, Compass, Mail } from 'lucide-react';
|
||||||
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
export async function generateMetadata({
|
||||||
|
params,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string }>;
|
||||||
|
}) {
|
||||||
|
const { locale } = await params;
|
||||||
|
return {
|
||||||
|
alternates: getPageAlternates(locale, '/docs/api'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export default async function ApiDocsPage({
|
export default async function ApiDocsPage({
|
||||||
params,
|
params,
|
||||||
|
|||||||
@@ -3,6 +3,20 @@ import { Header } from '@/components/Header';
|
|||||||
import { Footer } from '@/components/Footer';
|
import { Footer } from '@/components/Footer';
|
||||||
import Link from 'next/link';
|
import Link from 'next/link';
|
||||||
import { ArrowLeft, ArrowRight } from 'lucide-react';
|
import { ArrowLeft, ArrowRight } from 'lucide-react';
|
||||||
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
export async function generateMetadata({
|
||||||
|
params,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string }>;
|
||||||
|
}) {
|
||||||
|
const { locale } = await params;
|
||||||
|
return {
|
||||||
|
alternates: getPageAlternates(locale, '/docs/cli'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export default async function CliDocsPage({
|
export default async function CliDocsPage({
|
||||||
params,
|
params,
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ import Link from 'next/link';
|
|||||||
import { ArrowLeft, ArrowRight } from 'lucide-react';
|
import { ArrowLeft, ArrowRight } from 'lucide-react';
|
||||||
import { createDb, skills, sql } from '@skillhub/db';
|
import { createDb, skills, sql } from '@skillhub/db';
|
||||||
import { formatPromptSkillCount } from '@/lib/format-number';
|
import { formatPromptSkillCount } from '@/lib/format-number';
|
||||||
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
|
|
||||||
|
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
@@ -15,7 +17,7 @@ async function getSkillCount(): Promise<string> {
|
|||||||
const result = await db
|
const result = await db
|
||||||
.select({ count: sql<number>`count(*)::int` })
|
.select({ count: sql<number>`count(*)::int` })
|
||||||
.from(skills)
|
.from(skills)
|
||||||
.where(sql`${skills.isDuplicate} = false AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`);
|
.where(sql`${skills.isDuplicate} = false`);
|
||||||
return formatPromptSkillCount(result[0]?.count ?? 16000);
|
return formatPromptSkillCount(result[0]?.count ?? 16000);
|
||||||
} catch {
|
} catch {
|
||||||
return '16,000+';
|
return '16,000+';
|
||||||
@@ -56,6 +58,18 @@ Search for unfamiliar tech or complex tasks. Only install "Pass" security status
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
export async function generateMetadata({
|
||||||
|
params,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string }>;
|
||||||
|
}) {
|
||||||
|
const { locale } = await params;
|
||||||
|
return {
|
||||||
|
alternates: getPageAlternates(locale, '/docs/getting-started'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export default async function GettingStartedPage({
|
export default async function GettingStartedPage({
|
||||||
params,
|
params,
|
||||||
}: {
|
}: {
|
||||||
|
|||||||
@@ -3,6 +3,20 @@ import { Header } from '@/components/Header';
|
|||||||
import { Footer } from '@/components/Footer';
|
import { Footer } from '@/components/Footer';
|
||||||
import { BookOpen, Terminal, Code } from 'lucide-react';
|
import { BookOpen, Terminal, Code } from 'lucide-react';
|
||||||
import Link from 'next/link';
|
import Link from 'next/link';
|
||||||
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
export async function generateMetadata({
|
||||||
|
params,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string }>;
|
||||||
|
}) {
|
||||||
|
const { locale } = await params;
|
||||||
|
return {
|
||||||
|
alternates: getPageAlternates(locale, '/docs'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export default async function DocsPage({
|
export default async function DocsPage({
|
||||||
params,
|
params,
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ import { Footer } from '@/components/Footer';
|
|||||||
import { FavoritesList } from '@/components/FavoritesList';
|
import { FavoritesList } from '@/components/FavoritesList';
|
||||||
import { FavoritesSignIn } from '@/components/FavoritesSignIn';
|
import { FavoritesSignIn } from '@/components/FavoritesSignIn';
|
||||||
import { createDb, userQueries } from '@skillhub/db';
|
import { createDb, userQueries } from '@skillhub/db';
|
||||||
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
|
|
||||||
|
|
||||||
// Force dynamic rendering
|
// Force dynamic rendering
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
@@ -14,6 +16,18 @@ interface FavoritesPageProps {
|
|||||||
params: Promise<{ locale: string }>;
|
params: Promise<{ locale: string }>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
export async function generateMetadata({
|
||||||
|
params,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string }>;
|
||||||
|
}) {
|
||||||
|
const { locale } = await params;
|
||||||
|
return {
|
||||||
|
alternates: getPageAlternates(locale, '/favorites'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export default async function FavoritesPage({ params }: FavoritesPageProps) {
|
export default async function FavoritesPage({ params }: FavoritesPageProps) {
|
||||||
const { locale } = await params;
|
const { locale } = await params;
|
||||||
setRequestLocale(locale);
|
setRequestLocale(locale);
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ import { createDb, skillQueries } from '@skillhub/db';
|
|||||||
import { toPersianNumber } from '@/lib/format-number';
|
import { toPersianNumber } from '@/lib/format-number';
|
||||||
import { Pagination } from '@/components/BrowseFilters';
|
import { Pagination } from '@/components/BrowseFilters';
|
||||||
import { SkillCard } from '@/components/SkillCard';
|
import { SkillCard } from '@/components/SkillCard';
|
||||||
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
|
|
||||||
|
|
||||||
// Force dynamic rendering to fetch fresh data from database
|
// Force dynamic rendering to fetch fresh data from database
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
@@ -37,6 +39,18 @@ async function getFeaturedSkills(page: number, limit: number) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
export async function generateMetadata({
|
||||||
|
params,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string }>;
|
||||||
|
}) {
|
||||||
|
const { locale } = await params;
|
||||||
|
return {
|
||||||
|
alternates: getPageAlternates(locale, '/featured'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export default async function FeaturedPage({
|
export default async function FeaturedPage({
|
||||||
params,
|
params,
|
||||||
searchParams,
|
searchParams,
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ import { Clock, RefreshCw } from 'lucide-react';
|
|||||||
import { createDb, skillQueries } from '@skillhub/db';
|
import { createDb, skillQueries } from '@skillhub/db';
|
||||||
import { toPersianNumber } from '@/lib/format-number';
|
import { toPersianNumber } from '@/lib/format-number';
|
||||||
import { Pagination } from '@/components/BrowseFilters';
|
import { Pagination } from '@/components/BrowseFilters';
|
||||||
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
|
|
||||||
|
|
||||||
// Force dynamic rendering to fetch fresh data from database
|
// Force dynamic rendering to fetch fresh data from database
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
@@ -75,6 +77,18 @@ async function getTabCounts() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
export async function generateMetadata({
|
||||||
|
params,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string }>;
|
||||||
|
}) {
|
||||||
|
const { locale } = await params;
|
||||||
|
return {
|
||||||
|
alternates: getPageAlternates(locale, '/new'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export default async function NewSkillsPage({
|
export default async function NewSkillsPage({
|
||||||
params,
|
params,
|
||||||
searchParams,
|
searchParams,
|
||||||
|
|||||||
@@ -7,9 +7,23 @@ import { createDb, skillQueries } from '@skillhub/db';
|
|||||||
import { formatCompactNumber, toPersianNumber } from '@/lib/format-number';
|
import { formatCompactNumber, toPersianNumber } from '@/lib/format-number';
|
||||||
import { ExternalLink, Download, Package, Eye, GitFork, ArrowUpDown, FolderGit2 } from 'lucide-react';
|
import { ExternalLink, Download, Package, Eye, GitFork, ArrowUpDown, FolderGit2 } from 'lucide-react';
|
||||||
import Link from 'next/link';
|
import Link from 'next/link';
|
||||||
|
import type { Metadata } from 'next';
|
||||||
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
|
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
|
export async function generateMetadata({
|
||||||
|
params,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string; username: string }>;
|
||||||
|
}): Promise<Metadata> {
|
||||||
|
const { locale, username } = await params;
|
||||||
|
return {
|
||||||
|
title: `${decodeURIComponent(username)} | SkillHub`,
|
||||||
|
alternates: getPageAlternates(locale, `/owner/${username}`),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
const ITEMS_PER_PAGE = 24;
|
const ITEMS_PER_PAGE = 24;
|
||||||
|
|
||||||
type SortOption = 'popularity' | 'downloads' | 'stars';
|
type SortOption = 'popularity' | 'downloads' | 'stars';
|
||||||
@@ -183,8 +197,7 @@ export default async function OwnerPage({ params, searchParams }: OwnerPageProps
|
|||||||
<span className="text-sm text-text-secondary">{t('repo.filter')}:</span>
|
<span className="text-sm text-text-secondary">{t('repo.filter')}:</span>
|
||||||
<Link
|
<Link
|
||||||
href={buildUrl({ repo: '', page: 1 })}
|
href={buildUrl({ repo: '', page: 1 })}
|
||||||
className={`px-3 py-1 text-xs rounded-full transition-colors ${
|
className={`px-3 py-1 text-xs rounded-full transition-colors ${!activeRepo
|
||||||
!activeRepo
|
|
||||||
? 'bg-primary text-primary-foreground font-medium'
|
? 'bg-primary text-primary-foreground font-medium'
|
||||||
: 'bg-surface-elevated text-text-secondary hover:bg-surface-subtle border border-border'
|
: 'bg-surface-elevated text-text-secondary hover:bg-surface-subtle border border-border'
|
||||||
}`}
|
}`}
|
||||||
@@ -195,8 +208,7 @@ export default async function OwnerPage({ params, searchParams }: OwnerPageProps
|
|||||||
<Link
|
<Link
|
||||||
key={r.repo}
|
key={r.repo}
|
||||||
href={buildUrl({ repo: r.repo, page: 1 })}
|
href={buildUrl({ repo: r.repo, page: 1 })}
|
||||||
className={`px-3 py-1 text-xs rounded-full transition-colors ${
|
className={`px-3 py-1 text-xs rounded-full transition-colors ${activeRepo === r.repo
|
||||||
activeRepo === r.repo
|
|
||||||
? 'bg-primary text-primary-foreground font-medium'
|
? 'bg-primary text-primary-foreground font-medium'
|
||||||
: 'bg-surface-elevated text-text-secondary hover:bg-surface-subtle border border-border'
|
: 'bg-surface-elevated text-text-secondary hover:bg-surface-subtle border border-border'
|
||||||
}`}
|
}`}
|
||||||
@@ -227,8 +239,7 @@ export default async function OwnerPage({ params, searchParams }: OwnerPageProps
|
|||||||
<Link
|
<Link
|
||||||
key={opt.value}
|
key={opt.value}
|
||||||
href={buildUrl({ sort: opt.value, page: 1 })}
|
href={buildUrl({ sort: opt.value, page: 1 })}
|
||||||
className={`px-3 py-1.5 text-sm rounded-lg transition-colors ${
|
className={`px-3 py-1.5 text-sm rounded-lg transition-colors ${sort === opt.value
|
||||||
sort === opt.value
|
|
||||||
? 'bg-primary text-primary-foreground font-medium'
|
? 'bg-primary text-primary-foreground font-medium'
|
||||||
: 'bg-surface-elevated text-text-secondary hover:bg-surface-subtle border border-border'
|
: 'bg-surface-elevated text-text-secondary hover:bg-surface-subtle border border-border'
|
||||||
}`}
|
}`}
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ import { HeroSearch } from '@/components/HeroSearch';
|
|||||||
import { SkillCard } from '@/components/SkillCard';
|
import { SkillCard } from '@/components/SkillCard';
|
||||||
import { createDb, categoryQueries, skillQueries, skills, sql } from '@skillhub/db';
|
import { createDb, categoryQueries, skillQueries, skills, sql } from '@skillhub/db';
|
||||||
import { formatCompactNumber } from '@/lib/format-number';
|
import { formatCompactNumber } from '@/lib/format-number';
|
||||||
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
|
|
||||||
|
|
||||||
// Force dynamic rendering to fetch fresh data from database
|
// Force dynamic rendering to fetch fresh data from database
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
@@ -16,8 +18,8 @@ async function getStats() {
|
|||||||
try {
|
try {
|
||||||
const db = createDb();
|
const db = createDb();
|
||||||
|
|
||||||
// Browse-ready filter: exclude duplicates and aggregators
|
// Browse-ready filter: exclude duplicates (matches browseReadyFilter in queries.ts)
|
||||||
const browseReady = sql`${skills.isDuplicate} = false AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`;
|
const browseReady = sql`${skills.isDuplicate} = false`;
|
||||||
|
|
||||||
// Get total skills count (browse-ready, SKILL.md only)
|
// Get total skills count (browse-ready, SKILL.md only)
|
||||||
const skillsResult = await db
|
const skillsResult = await db
|
||||||
@@ -82,6 +84,18 @@ async function getFeaturedSkills() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
export async function generateMetadata({
|
||||||
|
params,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string }>;
|
||||||
|
}) {
|
||||||
|
const { locale } = await params;
|
||||||
|
return {
|
||||||
|
alternates: getPageAlternates(locale, '/'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export default async function HomePage({
|
export default async function HomePage({
|
||||||
params,
|
params,
|
||||||
}: {
|
}: {
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ import Link from 'next/link';
|
|||||||
import { Header } from '@/components/Header';
|
import { Header } from '@/components/Header';
|
||||||
import { Footer } from '@/components/Footer';
|
import { Footer } from '@/components/Footer';
|
||||||
import { Shield, Database, Cookie, Users, Lock, Mail, ArrowRight, ArrowLeft } from 'lucide-react';
|
import { Shield, Database, Cookie, Users, Lock, Mail, ArrowRight, ArrowLeft } from 'lucide-react';
|
||||||
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
|
|
||||||
|
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
@@ -15,6 +17,18 @@ const sectionIcons = {
|
|||||||
contact: Mail,
|
contact: Mail,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
export async function generateMetadata({
|
||||||
|
params,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string }>;
|
||||||
|
}) {
|
||||||
|
const { locale } = await params;
|
||||||
|
return {
|
||||||
|
alternates: getPageAlternates(locale, '/privacy'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export default async function PrivacyPage({
|
export default async function PrivacyPage({
|
||||||
params,
|
params,
|
||||||
}: {
|
}: {
|
||||||
|
|||||||
@@ -16,10 +16,32 @@ import { createDb, skillQueries } from '@skillhub/db';
|
|||||||
import { FORMAT_LABELS } from 'skillhub-core';
|
import { FORMAT_LABELS } from 'skillhub-core';
|
||||||
import { formatCompactNumber } from '@/lib/format-number';
|
import { formatCompactNumber } from '@/lib/format-number';
|
||||||
import { shouldCountView } from '@/lib/cache';
|
import { shouldCountView } from '@/lib/cache';
|
||||||
|
import type { Metadata } from 'next';
|
||||||
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
|
|
||||||
// Force dynamic rendering to fetch fresh data from database
|
// Force dynamic rendering to fetch fresh data from database
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
|
export async function generateMetadata({
|
||||||
|
params,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string; id: string[] }>;
|
||||||
|
}): Promise<Metadata> {
|
||||||
|
const { locale, id } = await params;
|
||||||
|
const skillId = id.join('/');
|
||||||
|
|
||||||
|
// Optionally fetch skill name for title
|
||||||
|
const dbSkill = await getSkill(skillId);
|
||||||
|
const title = dbSkill ? `${dbSkill.name} | SkillHub` : 'SkillHub';
|
||||||
|
const description = dbSkill ? dbSkill.description : undefined;
|
||||||
|
|
||||||
|
return {
|
||||||
|
title,
|
||||||
|
description,
|
||||||
|
alternates: getPageAlternates(locale, `/skill/${skillId}`),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
interface SkillPageProps {
|
interface SkillPageProps {
|
||||||
params: Promise<{ locale: string; id: string[] }>;
|
params: Promise<{ locale: string; id: string[] }>;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ import { getTranslations, setRequestLocale } from 'next-intl/server';
|
|||||||
import { Header } from '@/components/Header';
|
import { Header } from '@/components/Header';
|
||||||
import { Footer } from '@/components/Footer';
|
import { Footer } from '@/components/Footer';
|
||||||
import { Mail, Bitcoin, ExternalLink } from 'lucide-react';
|
import { Mail, Bitcoin, ExternalLink } from 'lucide-react';
|
||||||
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
|
|
||||||
|
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
@@ -9,6 +11,18 @@ interface SupportPageProps {
|
|||||||
params: Promise<{ locale: string }>;
|
params: Promise<{ locale: string }>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
export async function generateMetadata({
|
||||||
|
params,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string }>;
|
||||||
|
}) {
|
||||||
|
const { locale } = await params;
|
||||||
|
return {
|
||||||
|
alternates: getPageAlternates(locale, '/support'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export default async function SupportPage({ params }: SupportPageProps) {
|
export default async function SupportPage({ params }: SupportPageProps) {
|
||||||
const { locale } = await params;
|
const { locale } = await params;
|
||||||
setRequestLocale(locale);
|
setRequestLocale(locale);
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ import Link from 'next/link';
|
|||||||
import { Header } from '@/components/Header';
|
import { Header } from '@/components/Header';
|
||||||
import { Footer } from '@/components/Footer';
|
import { Footer } from '@/components/Footer';
|
||||||
import { FileText, UserCheck, AlertTriangle, Scale, Trash2, RefreshCw, ArrowRight, ArrowLeft } from 'lucide-react';
|
import { FileText, UserCheck, AlertTriangle, Scale, Trash2, RefreshCw, ArrowRight, ArrowLeft } from 'lucide-react';
|
||||||
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
|
|
||||||
|
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
@@ -15,6 +17,18 @@ const sectionIcons = {
|
|||||||
changes: RefreshCw,
|
changes: RefreshCw,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
export async function generateMetadata({
|
||||||
|
params,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string }>;
|
||||||
|
}) {
|
||||||
|
const { locale } = await params;
|
||||||
|
return {
|
||||||
|
alternates: getPageAlternates(locale, '/terms'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export default async function TermsPage({
|
export default async function TermsPage({
|
||||||
params,
|
params,
|
||||||
}: {
|
}: {
|
||||||
|
|||||||
@@ -34,8 +34,8 @@ export async function GET(request: NextRequest) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Browse-ready filter: exclude duplicates and aggregators
|
// Browse-ready filter: exclude duplicates (matches browseReadyFilter in queries.ts)
|
||||||
const browseReady = sql`${skills.isDuplicate} = false AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`;
|
const browseReady = sql`${skills.isDuplicate} = false`;
|
||||||
|
|
||||||
// Browse-ready skill stats (skills count + contributors)
|
// Browse-ready skill stats (skills count + contributors)
|
||||||
const statsResult = await db
|
const statsResult = await db
|
||||||
|
|||||||
@@ -1,17 +1,8 @@
|
|||||||
import type { MetadataRoute } from 'next';
|
import type { MetadataRoute } from 'next';
|
||||||
import { createDb, skillQueries, categoryQueries } from '@skillhub/db';
|
import { createDb, skillQueries, categoryQueries } from '@skillhub/db';
|
||||||
|
import { getCanonicalPath, primaryDomain as BASE_URL } from '@/lib/seo';
|
||||||
const BASE_URL =
|
|
||||||
process.env.NEXT_PUBLIC_APP_URL || 'https://skills.palebluedot.live';
|
|
||||||
|
|
||||||
const locales = ['en', 'fa'] as const;
|
const locales = ['en', 'fa'] as const;
|
||||||
const DEFAULT_LOCALE = 'en';
|
|
||||||
|
|
||||||
/** Get canonical URL path for a locale (no prefix for default locale, matching localePrefix: 'as-needed') */
|
|
||||||
function canonicalPath(locale: string, route: string): string {
|
|
||||||
if (locale === DEFAULT_LOCALE) return route || '/';
|
|
||||||
return `/${locale}${route}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
const staticRoutes = [
|
const staticRoutes = [
|
||||||
'',
|
'',
|
||||||
@@ -43,13 +34,13 @@ function makeEntry(
|
|||||||
}
|
}
|
||||||
): MetadataRoute.Sitemap[number] {
|
): MetadataRoute.Sitemap[number] {
|
||||||
return {
|
return {
|
||||||
url: `${BASE_URL}${canonicalPath(locale, route)}`,
|
url: `${BASE_URL}${getCanonicalPath(locale, route)}`,
|
||||||
lastModified: options?.lastModified,
|
lastModified: options?.lastModified,
|
||||||
changeFrequency: options?.changeFrequency,
|
changeFrequency: options?.changeFrequency,
|
||||||
priority: options?.priority,
|
priority: options?.priority,
|
||||||
alternates: {
|
alternates: {
|
||||||
languages: Object.fromEntries(
|
languages: Object.fromEntries(
|
||||||
locales.map((l) => [l, `${BASE_URL}${canonicalPath(l, route)}`])
|
locales.map((l) => [l, `${BASE_URL}${getCanonicalPath(l, route)}`])
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|||||||
50
apps/web/lib/seo.ts
Normal file
50
apps/web/lib/seo.ts
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
import { locales } from '@/i18n';
|
||||||
|
|
||||||
|
export const primaryDomain =
|
||||||
|
process.env.NEXT_PUBLIC_APP_URL || 'https://skills.palebluedot.live';
|
||||||
|
|
||||||
|
const DEFAULT_LOCALE = 'en';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get canonical URL path for a locale
|
||||||
|
* (no prefix for default locale, matching localePrefix: 'as-needed')
|
||||||
|
*/
|
||||||
|
export function getCanonicalPath(locale: string, route: string): string {
|
||||||
|
// Normalize route to always start with a slash and not end with a slash
|
||||||
|
let formattedRoute = route;
|
||||||
|
if (!formattedRoute.startsWith('/')) {
|
||||||
|
formattedRoute = `/${formattedRoute}`;
|
||||||
|
}
|
||||||
|
if (formattedRoute.length > 1 && formattedRoute.endsWith('/')) {
|
||||||
|
formattedRoute = formattedRoute.slice(0, -1);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (locale === DEFAULT_LOCALE) {
|
||||||
|
if (formattedRoute === '/') return '';
|
||||||
|
return formattedRoute;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (formattedRoute === '/') return `/${locale}`;
|
||||||
|
return `/${locale}${formattedRoute}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate Next.js alternates metadata for a specific route
|
||||||
|
* @param locale Current active locale
|
||||||
|
* @param route The route path (e.g. '/about', '/categories')
|
||||||
|
*/
|
||||||
|
export function getPageAlternates(locale: string, route: string) {
|
||||||
|
// Ensure we fall back to / if it's empty
|
||||||
|
const canonicalPath = getCanonicalPath(locale, route) || '/';
|
||||||
|
|
||||||
|
const languages: Record<string, string> = {};
|
||||||
|
|
||||||
|
for (const l of locales) {
|
||||||
|
languages[l] = `${primaryDomain}${getCanonicalPath(l, route) || '/'}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
canonical: `${primaryDomain}${canonicalPath}`,
|
||||||
|
languages,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -31,11 +31,11 @@ function getRawClientLocal() {
|
|||||||
type DB = PostgresJsDatabase<typeof schema>;
|
type DB = PostgresJsDatabase<typeof schema>;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Curation filter: excludes duplicates and aggregator-only skills.
|
* Curation filter: excludes duplicates.
|
||||||
* Skills with skill_type=NULL (newly crawled, not yet curated) are included
|
* Skills with skill_type=NULL (newly crawled, not yet curated) are included
|
||||||
* so they don't disappear between curate.mjs runs.
|
* so they don't disappear between curate.mjs runs.
|
||||||
*/
|
*/
|
||||||
const browseReadyFilter = sql`(${skills.isDuplicate} = false) AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`;
|
const browseReadyFilter = sql`(${skills.isDuplicate} = false)`;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Skill queries
|
* Skill queries
|
||||||
@@ -556,17 +556,38 @@ export const skillQueries = {
|
|||||||
skill: typeof skills.$inferInsert
|
skill: typeof skills.$inferInsert
|
||||||
): Promise<typeof skills.$inferSelect> => {
|
): Promise<typeof skills.$inferSelect> => {
|
||||||
// Check if commitSha changed (for cache invalidation)
|
// Check if commitSha changed (for cache invalidation)
|
||||||
|
// and if content_hash changed (for re-review mechanism)
|
||||||
let shouldClearCache = false;
|
let shouldClearCache = false;
|
||||||
if (skill.id && skill.commitSha) {
|
let shouldTriggerReReview = false;
|
||||||
|
if (skill.id) {
|
||||||
const existing = await db
|
const existing = await db
|
||||||
.select({ commitSha: skills.commitSha, cachedFiles: skills.cachedFiles })
|
.select({
|
||||||
|
commitSha: skills.commitSha,
|
||||||
|
cachedFiles: skills.cachedFiles,
|
||||||
|
contentHash: skills.contentHash,
|
||||||
|
reviewStatus: skills.reviewStatus,
|
||||||
|
})
|
||||||
.from(skills)
|
.from(skills)
|
||||||
.where(eq(skills.id, skill.id))
|
.where(eq(skills.id, skill.id))
|
||||||
.limit(1);
|
.limit(1);
|
||||||
|
|
||||||
if (existing[0] && existing[0].cachedFiles && existing[0].commitSha !== skill.commitSha) {
|
if (existing[0]) {
|
||||||
|
// Cache invalidation: clear cachedFiles when commitSha changes
|
||||||
|
if (skill.commitSha && existing[0].cachedFiles && existing[0].commitSha !== skill.commitSha) {
|
||||||
shouldClearCache = true;
|
shouldClearCache = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Re-review mechanism (Phase 6.4): when content_hash changes and skill
|
||||||
|
// was previously reviewed, mark it for re-review
|
||||||
|
if (
|
||||||
|
skill.contentHash &&
|
||||||
|
existing[0].contentHash &&
|
||||||
|
existing[0].contentHash !== skill.contentHash &&
|
||||||
|
(existing[0].reviewStatus === 'verified' || existing[0].reviewStatus === 'ai-reviewed')
|
||||||
|
) {
|
||||||
|
shouldTriggerReReview = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const result = await db
|
const result = await db
|
||||||
@@ -605,6 +626,9 @@ export const skillQueries = {
|
|||||||
// which only updates it when content-related columns actually change
|
// which only updates it when content-related columns actually change
|
||||||
// Clear cachedFiles if commitSha changed
|
// Clear cachedFiles if commitSha changed
|
||||||
...(shouldClearCache ? { cachedFiles: null } : {}),
|
...(shouldClearCache ? { cachedFiles: null } : {}),
|
||||||
|
// Re-review mechanism (Phase 6.4): mark previously reviewed skills for re-review
|
||||||
|
// when their content changes. Only affects 'verified' and 'ai-reviewed' skills.
|
||||||
|
...(shouldTriggerReReview ? { reviewStatus: 'needs-re-review' } : {}),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
.returning();
|
.returning();
|
||||||
@@ -1172,7 +1196,7 @@ export async function runPostCrawlCuration(db: DB): Promise<{ classified: number
|
|||||||
`);
|
`);
|
||||||
results.classified = (classResult as unknown as { rowCount: number }).rowCount ?? 0;
|
results.classified = (classResult as unknown as { rowCount: number }).rowCount ?? 0;
|
||||||
|
|
||||||
// Step 4: Mark duplicates by content_hash (keep canonical = most stars)
|
// Step 4: Mark duplicates by content_hash (keep canonical = most stars, prioritizing standalone over aggregator)
|
||||||
const dupResult = await db.execute(sql`
|
const dupResult = await db.execute(sql`
|
||||||
UPDATE skills s SET is_duplicate = true
|
UPDATE skills s SET is_duplicate = true
|
||||||
WHERE s.is_blocked = false
|
WHERE s.is_blocked = false
|
||||||
@@ -1183,8 +1207,17 @@ export async function runPostCrawlCuration(db: DB): Promise<{ classified: number
|
|||||||
WHERE s2.content_hash = s.content_hash
|
WHERE s2.content_hash = s.content_hash
|
||||||
AND s2.is_blocked = false
|
AND s2.is_blocked = false
|
||||||
AND s2.id != s.id
|
AND s2.id != s.id
|
||||||
AND (s2.github_stars > s.github_stars
|
AND (
|
||||||
OR (s2.github_stars = s.github_stars AND s2.indexed_at < s.indexed_at))
|
(s2.skill_type != 'aggregator' AND s.skill_type = 'aggregator')
|
||||||
|
OR (
|
||||||
|
(s2.skill_type = 'aggregator') = (s.skill_type = 'aggregator')
|
||||||
|
AND (
|
||||||
|
s2.github_stars > s.github_stars
|
||||||
|
OR (s2.github_stars = s.github_stars AND s2.github_forks > s.github_forks)
|
||||||
|
OR (s2.github_stars = s.github_stars AND s2.github_forks = s.github_forks AND s2.created_at < s.created_at)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
)
|
)
|
||||||
`);
|
`);
|
||||||
results.duplicates = (dupResult as unknown as { rowCount: number }).rowCount ?? 0;
|
results.duplicates = (dupResult as unknown as { rowCount: number }).rowCount ?? 0;
|
||||||
@@ -1198,7 +1231,6 @@ export async function runPostCrawlCuration(db: DB): Promise<{ classified: number
|
|||||||
JOIN skills s ON sc.skill_id = s.id
|
JOIN skills s ON sc.skill_id = s.id
|
||||||
WHERE s.is_blocked = false
|
WHERE s.is_blocked = false
|
||||||
AND s.is_duplicate = false
|
AND s.is_duplicate = false
|
||||||
AND (s.skill_type IS NULL OR s.skill_type != 'aggregator')
|
|
||||||
GROUP BY sc.category_id
|
GROUP BY sc.category_id
|
||||||
) sub
|
) sub
|
||||||
WHERE c.id = sub.category_id
|
WHERE c.id = sub.category_id
|
||||||
|
|||||||
@@ -79,6 +79,7 @@ export const skills = pgTable(
|
|||||||
isDuplicate: boolean('is_duplicate').default(false),
|
isDuplicate: boolean('is_duplicate').default(false),
|
||||||
canonicalSkillId: text('canonical_skill_id'), // points to the "original" if this is a duplicate
|
canonicalSkillId: text('canonical_skill_id'), // points to the "original" if this is a duplicate
|
||||||
repoSkillCount: integer('repo_skill_count'), // cached count of skills in same repo
|
repoSkillCount: integer('repo_skill_count'), // cached count of skills in same repo
|
||||||
|
reviewStatus: text('review_status').default('unreviewed'), // unreviewed → auto-scored → ai-reviewed → verified (or needs-re-review)
|
||||||
|
|
||||||
// Content (cached)
|
// Content (cached)
|
||||||
contentHash: text('content_hash'),
|
contentHash: text('content_hash'),
|
||||||
|
|||||||
459
scripts/curation/batch-score.mjs
Normal file
459
scripts/curation/batch-score.mjs
Normal file
@@ -0,0 +1,459 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* Phase 5: Batch Quality Score
|
||||||
|
*
|
||||||
|
* Calculates quality scores for browse-ready skills using the same logic
|
||||||
|
* as SkillAnalyzer.calculateQuality() from services/indexer/src/analyzer.ts.
|
||||||
|
*
|
||||||
|
* Updates: quality_score, quality_details, review_status → 'auto-scored'
|
||||||
|
*
|
||||||
|
* Usage:
|
||||||
|
* DATABASE_URL=postgres://... node scripts/curation/batch-score.mjs
|
||||||
|
*
|
||||||
|
* Options:
|
||||||
|
* --dry-run Show what would change without writing
|
||||||
|
* --batch-size=N Process N skills per batch (default 100)
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { createRequire } from 'module';
|
||||||
|
import { resolve, dirname } from 'path';
|
||||||
|
import { fileURLToPath, pathToFileURL } from 'url';
|
||||||
|
|
||||||
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||||
|
|
||||||
|
// ─── Find pg module ───
|
||||||
|
let pg;
|
||||||
|
const tryPaths = [
|
||||||
|
resolve(__dirname, '../..', 'package.json'),
|
||||||
|
'/tmp/package.json',
|
||||||
|
process.env.APPDATA ? resolve(process.env.APPDATA, '..', 'Local', 'Temp', 'package.json') : null,
|
||||||
|
].filter(Boolean);
|
||||||
|
for (const p of tryPaths) {
|
||||||
|
try { pg = createRequire(p)('pg'); break; } catch {}
|
||||||
|
}
|
||||||
|
if (!pg) { console.error('pg not found. Run: npm install pg'); process.exit(1); }
|
||||||
|
|
||||||
|
// ─── Find skillhub-core (ESM-only — use direct path import) ───
|
||||||
|
let parseSkillMd, parseGenericInstructionFile, validateSkill, scanSecurity;
|
||||||
|
const corePaths = [
|
||||||
|
resolve(__dirname, '../../packages/core/dist/index.js'),
|
||||||
|
resolve(__dirname, '../../node_modules/skillhub-core/dist/index.js'),
|
||||||
|
resolve(__dirname, '../../services/indexer/node_modules/skillhub-core/dist/index.js'),
|
||||||
|
];
|
||||||
|
for (const p of corePaths) {
|
||||||
|
try {
|
||||||
|
const core = await import(pathToFileURL(p).href);
|
||||||
|
parseSkillMd = core.parseSkillMd;
|
||||||
|
parseGenericInstructionFile = core.parseGenericInstructionFile;
|
||||||
|
validateSkill = core.validateSkill;
|
||||||
|
scanSecurity = core.scanSecurity;
|
||||||
|
break;
|
||||||
|
} catch {}
|
||||||
|
}
|
||||||
|
if (!parseSkillMd) { console.error('skillhub-core not found. Run: pnpm build'); process.exit(1); }
|
||||||
|
|
||||||
|
// ─── Config ───
|
||||||
|
const DATABASE_URL = process.env.DATABASE_URL || 'postgresql://postgres:postgres@localhost:5432/skillhub';
|
||||||
|
const DRY_RUN = process.argv.includes('--dry-run');
|
||||||
|
const batchArg = process.argv.find(a => a.startsWith('--batch-size='));
|
||||||
|
const BATCH_SIZE = batchArg ? parseInt(batchArg.split('=')[1]) : 100;
|
||||||
|
|
||||||
|
// ─── Helpers ───
|
||||||
|
const n = v => Number(v ?? 0).toLocaleString('en-US');
|
||||||
|
|
||||||
|
function progress(current, total) {
|
||||||
|
const pct = ((current / total) * 100).toFixed(1);
|
||||||
|
process.stdout.write(`\r Processing ${n(current)} / ${n(total)} (${pct}%)`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Database ───
|
||||||
|
let client;
|
||||||
|
|
||||||
|
async function connect() {
|
||||||
|
client = new pg.Client({
|
||||||
|
connectionString: DATABASE_URL,
|
||||||
|
ssl: false,
|
||||||
|
connectionTimeoutMillis: 15000,
|
||||||
|
query_timeout: 600000,
|
||||||
|
keepAlive: true,
|
||||||
|
});
|
||||||
|
await client.connect();
|
||||||
|
console.log('Connected to database');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function query(sql, params = []) {
|
||||||
|
return client.query(sql, params);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Quality scoring (mirrors SkillAnalyzer.calculateQuality) ───
|
||||||
|
|
||||||
|
function scoreDocumentation(skill, scripts, references) {
|
||||||
|
let score = 0;
|
||||||
|
|
||||||
|
// Has description (required)
|
||||||
|
if (skill.metadata.description && skill.metadata.description.length > 20) {
|
||||||
|
score += 20;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Content length and structure
|
||||||
|
const contentLength = skill.content.length;
|
||||||
|
if (contentLength > 500) score += 15;
|
||||||
|
else if (contentLength > 200) score += 10;
|
||||||
|
else if (contentLength > 50) score += 5;
|
||||||
|
|
||||||
|
// Has headers (good structure)
|
||||||
|
const headerCount = (skill.content.match(/^#+\s/gm) || []).length;
|
||||||
|
if (headerCount >= 3) score += 15;
|
||||||
|
else if (headerCount >= 1) score += 10;
|
||||||
|
|
||||||
|
// Has code examples
|
||||||
|
if (skill.content.includes('```')) {
|
||||||
|
score += 15;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Has version
|
||||||
|
if (skill.metadata.version) {
|
||||||
|
score += 10;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Has license
|
||||||
|
if (skill.metadata.license) {
|
||||||
|
score += 5;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Has compatibility info
|
||||||
|
if (skill.metadata.compatibility?.platforms?.length) {
|
||||||
|
score += 10;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Has scripts
|
||||||
|
if (scripts.length > 0) {
|
||||||
|
score += 5;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Has references
|
||||||
|
if (references.length > 0) {
|
||||||
|
score += 5;
|
||||||
|
}
|
||||||
|
|
||||||
|
return Math.min(100, score);
|
||||||
|
}
|
||||||
|
|
||||||
|
function scoreMaintenance(repoMeta) {
|
||||||
|
let score = 0;
|
||||||
|
|
||||||
|
// Check last update time
|
||||||
|
const lastUpdate = new Date(repoMeta.updatedAt);
|
||||||
|
const daysSinceUpdate = (Date.now() - lastUpdate.getTime()) / (1000 * 60 * 60 * 24);
|
||||||
|
|
||||||
|
if (daysSinceUpdate < 30) score += 40;
|
||||||
|
else if (daysSinceUpdate < 90) score += 30;
|
||||||
|
else if (daysSinceUpdate < 180) score += 20;
|
||||||
|
else if (daysSinceUpdate < 365) score += 10;
|
||||||
|
|
||||||
|
// Has license
|
||||||
|
if (repoMeta.license) {
|
||||||
|
score += 20;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Has description
|
||||||
|
if (repoMeta.description) {
|
||||||
|
score += 10;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Has topics (always [] since not in DB — will miss ~10 points)
|
||||||
|
if (repoMeta.topics.length > 0) {
|
||||||
|
score += 10;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Activity level (forks indicate usage)
|
||||||
|
if (repoMeta.forks >= 10) score += 20;
|
||||||
|
else if (repoMeta.forks >= 5) score += 15;
|
||||||
|
else if (repoMeta.forks >= 1) score += 10;
|
||||||
|
|
||||||
|
return Math.min(100, score);
|
||||||
|
}
|
||||||
|
|
||||||
|
function scorePopularity(repoMeta) {
|
||||||
|
const stars = repoMeta.stars;
|
||||||
|
const forks = repoMeta.forks;
|
||||||
|
|
||||||
|
let score = 0;
|
||||||
|
|
||||||
|
if (stars >= 1000) score += 50;
|
||||||
|
else if (stars >= 100) score += 40;
|
||||||
|
else if (stars >= 50) score += 30;
|
||||||
|
else if (stars >= 10) score += 20;
|
||||||
|
else if (stars >= 5) score += 10;
|
||||||
|
else if (stars >= 1) score += 5;
|
||||||
|
|
||||||
|
if (forks >= 50) score += 30;
|
||||||
|
else if (forks >= 10) score += 20;
|
||||||
|
else if (forks >= 5) score += 15;
|
||||||
|
else if (forks >= 1) score += 10;
|
||||||
|
|
||||||
|
// Bonus for relevant topics (always [] from DB — will miss ~20 points)
|
||||||
|
const relevantTopics = ['ai', 'agent', 'skill', 'claude', 'copilot', 'codex', 'llm'];
|
||||||
|
const hasRelevantTopic = repoMeta.topics.some(t =>
|
||||||
|
relevantTopics.some(rt => t.toLowerCase().includes(rt))
|
||||||
|
);
|
||||||
|
if (hasRelevantTopic) {
|
||||||
|
score += 20;
|
||||||
|
}
|
||||||
|
|
||||||
|
return Math.min(100, score);
|
||||||
|
}
|
||||||
|
|
||||||
|
function calculateQuality(skill, repoMeta, securityScore, validation, scripts, references) {
|
||||||
|
const factors = [];
|
||||||
|
|
||||||
|
// Documentation quality (30% weight)
|
||||||
|
const docScore = scoreDocumentation(skill, scripts, references);
|
||||||
|
factors.push({ name: 'documentation', score: docScore, weight: 0.3 });
|
||||||
|
|
||||||
|
// Maintenance signals (25% weight)
|
||||||
|
const maintScore = scoreMaintenance(repoMeta);
|
||||||
|
factors.push({ name: 'maintenance', score: maintScore, weight: 0.25 });
|
||||||
|
|
||||||
|
// Popularity (20% weight)
|
||||||
|
const popScore = scorePopularity(repoMeta);
|
||||||
|
factors.push({ name: 'popularity', score: popScore, weight: 0.2 });
|
||||||
|
|
||||||
|
// Security (15% weight)
|
||||||
|
factors.push({ name: 'security', score: securityScore, weight: 0.15 });
|
||||||
|
|
||||||
|
// Validation (10% weight)
|
||||||
|
const valScore = validation.isValid ? 100 : Math.max(0, 100 - validation.errors.length * 20);
|
||||||
|
factors.push({ name: 'validation', score: valScore, weight: 0.1 });
|
||||||
|
|
||||||
|
// Calculate weighted overall score
|
||||||
|
const overall = Math.round(
|
||||||
|
factors.reduce((sum, f) => sum + f.score * f.weight, 0)
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
overall,
|
||||||
|
documentation: docScore,
|
||||||
|
maintenance: maintScore,
|
||||||
|
popularity: popScore,
|
||||||
|
factors,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Extract scripts/references from cached_files ───
|
||||||
|
function extractScripts(cachedFiles) {
|
||||||
|
if (!cachedFiles || !cachedFiles.items) return [];
|
||||||
|
return cachedFiles.items
|
||||||
|
.filter(item =>
|
||||||
|
!item.isBinary &&
|
||||||
|
item.name !== 'SKILL.md' &&
|
||||||
|
(item.name.endsWith('.sh') ||
|
||||||
|
item.name.endsWith('.py') ||
|
||||||
|
item.name.endsWith('.js') ||
|
||||||
|
item.name.endsWith('.ts') ||
|
||||||
|
item.name.endsWith('.ps1') ||
|
||||||
|
item.name.endsWith('.bat') ||
|
||||||
|
item.name.endsWith('.rb'))
|
||||||
|
)
|
||||||
|
.map(item => ({ name: item.name, content: item.content }));
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractReferences(cachedFiles) {
|
||||||
|
if (!cachedFiles || !cachedFiles.items) return [];
|
||||||
|
return cachedFiles.items
|
||||||
|
.filter(item =>
|
||||||
|
!item.isBinary &&
|
||||||
|
item.name !== 'SKILL.md' &&
|
||||||
|
(item.name.endsWith('.md') || item.name.endsWith('.txt'))
|
||||||
|
)
|
||||||
|
.map(item => ({ name: item.name, content: item.content }));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Main ───
|
||||||
|
async function main() {
|
||||||
|
const t0 = Date.now();
|
||||||
|
await connect();
|
||||||
|
|
||||||
|
console.log(`\n${'='.repeat(70)}`);
|
||||||
|
console.log(' BATCH QUALITY SCORE (Phase 5)');
|
||||||
|
console.log(`${'='.repeat(70)}`);
|
||||||
|
if (DRY_RUN) console.log('\n *** DRY RUN MODE — no changes will be made ***\n');
|
||||||
|
|
||||||
|
// Count skills to score
|
||||||
|
const countResult = await query(`
|
||||||
|
SELECT COUNT(*)::int AS total
|
||||||
|
FROM skills
|
||||||
|
WHERE is_blocked = false
|
||||||
|
AND is_duplicate = false
|
||||||
|
AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection'))
|
||||||
|
AND quality_score IS NULL
|
||||||
|
AND raw_content IS NOT NULL
|
||||||
|
`);
|
||||||
|
const total = countResult.rows[0].total;
|
||||||
|
console.log(` Skills to score: ${n(total)}`);
|
||||||
|
console.log(` Batch size: ${BATCH_SIZE}`);
|
||||||
|
|
||||||
|
if (total === 0) {
|
||||||
|
console.log(' Nothing to do — all browse-ready skills already scored');
|
||||||
|
await client.end().catch(() => {});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (DRY_RUN) {
|
||||||
|
const sample = await query(`
|
||||||
|
SELECT id, LEFT(name, 50) AS name, github_stars
|
||||||
|
FROM skills
|
||||||
|
WHERE is_blocked = false
|
||||||
|
AND is_duplicate = false
|
||||||
|
AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection'))
|
||||||
|
AND quality_score IS NULL
|
||||||
|
AND raw_content IS NOT NULL
|
||||||
|
ORDER BY github_stars DESC NULLS LAST
|
||||||
|
LIMIT 5
|
||||||
|
`);
|
||||||
|
console.log('\n Sample skills that would be scored:');
|
||||||
|
for (const r of sample.rows) {
|
||||||
|
console.log(` [${n(r.github_stars)}★] ${r.id} — ${r.name}`);
|
||||||
|
}
|
||||||
|
console.log(`\n [DRY RUN] Would score ${n(total)} skills`);
|
||||||
|
await client.end().catch(() => {});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process in batches
|
||||||
|
let processed = 0;
|
||||||
|
let errorCount = 0;
|
||||||
|
let totalScore = 0;
|
||||||
|
const scoreBuckets = { excellent: 0, good: 0, fair: 0, poor: 0 };
|
||||||
|
|
||||||
|
while (processed < total) {
|
||||||
|
const batch = await query(`
|
||||||
|
SELECT id, raw_content, cached_files, source_format,
|
||||||
|
github_stars, github_forks, license, description,
|
||||||
|
updated_at, version, compatibility, security_score
|
||||||
|
FROM skills
|
||||||
|
WHERE is_blocked = false
|
||||||
|
AND is_duplicate = false
|
||||||
|
AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection'))
|
||||||
|
AND quality_score IS NULL
|
||||||
|
AND raw_content IS NOT NULL
|
||||||
|
ORDER BY github_stars DESC NULLS LAST
|
||||||
|
LIMIT $1
|
||||||
|
`, [BATCH_SIZE]);
|
||||||
|
|
||||||
|
if (batch.rows.length === 0) break;
|
||||||
|
|
||||||
|
for (const row of batch.rows) {
|
||||||
|
try {
|
||||||
|
// Parse the skill content
|
||||||
|
const sourceFormat = row.source_format || 'skill.md';
|
||||||
|
let skill;
|
||||||
|
if (sourceFormat === 'skill.md') {
|
||||||
|
skill = parseSkillMd(row.raw_content);
|
||||||
|
} else {
|
||||||
|
skill = parseGenericInstructionFile(row.raw_content, sourceFormat, {
|
||||||
|
name: row.description?.split(/\s+/).slice(0, 3).join('-') || 'skill',
|
||||||
|
description: row.description,
|
||||||
|
owner: '',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate
|
||||||
|
const validation = sourceFormat === 'skill.md'
|
||||||
|
? validateSkill(skill)
|
||||||
|
: { isValid: true, errors: [], warnings: [] };
|
||||||
|
|
||||||
|
// Construct pseudo-repoMeta from DB fields
|
||||||
|
const repoMeta = {
|
||||||
|
stars: row.github_stars || 0,
|
||||||
|
forks: row.github_forks || 0,
|
||||||
|
license: row.license || null,
|
||||||
|
description: row.description || null,
|
||||||
|
updatedAt: row.updated_at ? row.updated_at.toISOString() : new Date().toISOString(),
|
||||||
|
defaultBranch: 'main',
|
||||||
|
topics: [], // NOT in DB — popularity factor will miss ~20 points
|
||||||
|
};
|
||||||
|
|
||||||
|
// Use already-computed security score (from Phase 4), default to 100 if not scanned
|
||||||
|
const securityScore = row.security_score ?? 100;
|
||||||
|
|
||||||
|
// Extract scripts/references from cached_files
|
||||||
|
const scripts = extractScripts(row.cached_files);
|
||||||
|
const references = extractReferences(row.cached_files);
|
||||||
|
|
||||||
|
// Calculate quality
|
||||||
|
const quality = calculateQuality(skill, repoMeta, securityScore, validation, scripts, references);
|
||||||
|
|
||||||
|
// Build quality_details for JSONB storage
|
||||||
|
const qualityDetails = {
|
||||||
|
documentation: quality.documentation,
|
||||||
|
maintenance: quality.maintenance,
|
||||||
|
popularity: quality.popularity,
|
||||||
|
factors: quality.factors,
|
||||||
|
};
|
||||||
|
|
||||||
|
await query(`
|
||||||
|
UPDATE skills
|
||||||
|
SET quality_score = $1,
|
||||||
|
quality_details = $2,
|
||||||
|
review_status = CASE
|
||||||
|
WHEN review_status IS NULL OR review_status = 'unreviewed'
|
||||||
|
THEN 'auto-scored'
|
||||||
|
ELSE review_status
|
||||||
|
END
|
||||||
|
WHERE id = $3
|
||||||
|
`, [quality.overall, JSON.stringify(qualityDetails), row.id]);
|
||||||
|
|
||||||
|
totalScore += quality.overall;
|
||||||
|
if (quality.overall >= 70) scoreBuckets.excellent++;
|
||||||
|
else if (quality.overall >= 50) scoreBuckets.good++;
|
||||||
|
else if (quality.overall >= 30) scoreBuckets.fair++;
|
||||||
|
else scoreBuckets.poor++;
|
||||||
|
} catch (err) {
|
||||||
|
errorCount++;
|
||||||
|
if (errorCount <= 5) {
|
||||||
|
console.error(`\n Error scoring ${row.id}: ${err.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
processed++;
|
||||||
|
if (processed % 100 === 0 || processed === total) {
|
||||||
|
progress(processed, total);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('\n'); // Clear progress line
|
||||||
|
|
||||||
|
// Summary
|
||||||
|
const scored = processed - errorCount;
|
||||||
|
const avgScore = scored > 0 ? (totalScore / scored).toFixed(1) : 0;
|
||||||
|
console.log(`
|
||||||
|
┌─────────────────────────────────────────────────────┐
|
||||||
|
│ QUALITY SCORE SUMMARY │
|
||||||
|
├─────────────────────────────────────────────────────┤
|
||||||
|
│ Total scored: ${n(scored).padStart(8)} │
|
||||||
|
│ Errors (skipped): ${n(errorCount).padStart(8)} │
|
||||||
|
├─────────────────────────────────────────────────────┤
|
||||||
|
│ Excellent (70+): ${n(scoreBuckets.excellent).padStart(8)} │
|
||||||
|
│ Good (50-69): ${n(scoreBuckets.good).padStart(8)} │
|
||||||
|
│ Fair (30-49): ${n(scoreBuckets.fair).padStart(8)} │
|
||||||
|
│ Poor (<30): ${n(scoreBuckets.poor).padStart(8)} │
|
||||||
|
├─────────────────────────────────────────────────────┤
|
||||||
|
│ Avg quality score: ${avgScore.toString().padStart(8)} │
|
||||||
|
└─────────────────────────────────────────────────────┘
|
||||||
|
`);
|
||||||
|
|
||||||
|
console.log(' Note: repoMeta.topics not in DB — popularity score misses ~20 points');
|
||||||
|
|
||||||
|
const dur = ((Date.now() - t0) / 1000).toFixed(1);
|
||||||
|
console.log(`\nCompleted in ${dur}s`);
|
||||||
|
|
||||||
|
await client.end().catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch(async e => {
|
||||||
|
console.error('FAILED:', e.message || e);
|
||||||
|
await client?.end().catch(() => {});
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
240
scripts/curation/batch-security.mjs
Normal file
240
scripts/curation/batch-security.mjs
Normal file
@@ -0,0 +1,240 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* Phase 4: Batch Security Scan
|
||||||
|
*
|
||||||
|
* Scans browse-ready skills for security issues using scanSecurity() from skillhub-core.
|
||||||
|
* Updates: security_score, security_status, last_scanned, review_status → 'auto-scored'
|
||||||
|
*
|
||||||
|
* Usage:
|
||||||
|
* DATABASE_URL=postgres://... node scripts/curation/batch-security.mjs
|
||||||
|
*
|
||||||
|
* Options:
|
||||||
|
* --dry-run Show what would change without writing
|
||||||
|
* --batch-size=N Process N skills per batch (default 100)
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { createRequire } from 'module';
|
||||||
|
import { resolve, dirname } from 'path';
|
||||||
|
import { fileURLToPath, pathToFileURL } from 'url';
|
||||||
|
|
||||||
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||||
|
|
||||||
|
// ─── Find pg module ───
|
||||||
|
let pg;
|
||||||
|
const tryPaths = [
|
||||||
|
resolve(__dirname, '../..', 'package.json'),
|
||||||
|
'/tmp/package.json',
|
||||||
|
process.env.APPDATA ? resolve(process.env.APPDATA, '..', 'Local', 'Temp', 'package.json') : null,
|
||||||
|
].filter(Boolean);
|
||||||
|
for (const p of tryPaths) {
|
||||||
|
try { pg = createRequire(p)('pg'); break; } catch {}
|
||||||
|
}
|
||||||
|
if (!pg) { console.error('pg not found. Run: npm install pg'); process.exit(1); }
|
||||||
|
|
||||||
|
// ─── Find skillhub-core (ESM-only — use direct path import) ───
|
||||||
|
let scanSecurity;
|
||||||
|
const corePaths = [
|
||||||
|
resolve(__dirname, '../../packages/core/dist/index.js'),
|
||||||
|
resolve(__dirname, '../../node_modules/skillhub-core/dist/index.js'),
|
||||||
|
resolve(__dirname, '../../services/indexer/node_modules/skillhub-core/dist/index.js'),
|
||||||
|
];
|
||||||
|
for (const p of corePaths) {
|
||||||
|
try {
|
||||||
|
const core = await import(pathToFileURL(p).href);
|
||||||
|
scanSecurity = core.scanSecurity;
|
||||||
|
break;
|
||||||
|
} catch {}
|
||||||
|
}
|
||||||
|
if (!scanSecurity) { console.error('skillhub-core not found. Run: pnpm build'); process.exit(1); }
|
||||||
|
|
||||||
|
// ─── Config ───
|
||||||
|
const DATABASE_URL = process.env.DATABASE_URL || 'postgresql://postgres:postgres@localhost:5432/skillhub';
|
||||||
|
const DRY_RUN = process.argv.includes('--dry-run');
|
||||||
|
const batchArg = process.argv.find(a => a.startsWith('--batch-size='));
|
||||||
|
const BATCH_SIZE = batchArg ? parseInt(batchArg.split('=')[1]) : 100;
|
||||||
|
|
||||||
|
// ─── Helpers ───
|
||||||
|
const n = v => Number(v ?? 0).toLocaleString('en-US');
|
||||||
|
|
||||||
|
function progress(current, total) {
|
||||||
|
const pct = ((current / total) * 100).toFixed(1);
|
||||||
|
process.stdout.write(`\r Processing ${n(current)} / ${n(total)} (${pct}%)`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Database ───
|
||||||
|
let client;
|
||||||
|
|
||||||
|
async function connect() {
|
||||||
|
client = new pg.Client({
|
||||||
|
connectionString: DATABASE_URL,
|
||||||
|
ssl: false,
|
||||||
|
connectionTimeoutMillis: 15000,
|
||||||
|
query_timeout: 600000,
|
||||||
|
keepAlive: true,
|
||||||
|
});
|
||||||
|
await client.connect();
|
||||||
|
console.log('Connected to database');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function query(sql, params = []) {
|
||||||
|
return client.query(sql, params);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Extract scripts from cached_files ───
|
||||||
|
function extractScripts(cachedFiles) {
|
||||||
|
if (!cachedFiles || !cachedFiles.items) return [];
|
||||||
|
return cachedFiles.items
|
||||||
|
.filter(item =>
|
||||||
|
!item.isBinary &&
|
||||||
|
item.name !== 'SKILL.md' &&
|
||||||
|
(item.name.endsWith('.sh') ||
|
||||||
|
item.name.endsWith('.py') ||
|
||||||
|
item.name.endsWith('.js') ||
|
||||||
|
item.name.endsWith('.ts') ||
|
||||||
|
item.name.endsWith('.ps1') ||
|
||||||
|
item.name.endsWith('.bat') ||
|
||||||
|
item.name.endsWith('.rb'))
|
||||||
|
)
|
||||||
|
.map(item => ({ name: item.name, content: item.content }));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Main ───
|
||||||
|
async function main() {
|
||||||
|
const t0 = Date.now();
|
||||||
|
await connect();
|
||||||
|
|
||||||
|
console.log(`\n${'='.repeat(70)}`);
|
||||||
|
console.log(' BATCH SECURITY SCAN (Phase 4)');
|
||||||
|
console.log(`${'='.repeat(70)}`);
|
||||||
|
if (DRY_RUN) console.log('\n *** DRY RUN MODE — no changes will be made ***\n');
|
||||||
|
|
||||||
|
// Count skills to scan
|
||||||
|
const countResult = await query(`
|
||||||
|
SELECT COUNT(*)::int AS total
|
||||||
|
FROM skills
|
||||||
|
WHERE is_blocked = false
|
||||||
|
AND is_duplicate = false
|
||||||
|
AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection'))
|
||||||
|
AND security_status IS NULL
|
||||||
|
AND raw_content IS NOT NULL
|
||||||
|
`);
|
||||||
|
const total = countResult.rows[0].total;
|
||||||
|
console.log(` Skills to scan: ${n(total)}`);
|
||||||
|
console.log(` Batch size: ${BATCH_SIZE}`);
|
||||||
|
|
||||||
|
if (total === 0) {
|
||||||
|
console.log(' Nothing to do — all browse-ready skills already scanned');
|
||||||
|
await client.end().catch(() => {});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (DRY_RUN) {
|
||||||
|
// Show sample
|
||||||
|
const sample = await query(`
|
||||||
|
SELECT id, LEFT(name, 50) AS name
|
||||||
|
FROM skills
|
||||||
|
WHERE is_blocked = false
|
||||||
|
AND is_duplicate = false
|
||||||
|
AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection'))
|
||||||
|
AND security_status IS NULL
|
||||||
|
AND raw_content IS NOT NULL
|
||||||
|
LIMIT 5
|
||||||
|
`);
|
||||||
|
console.log('\n Sample skills that would be scanned:');
|
||||||
|
for (const r of sample.rows) {
|
||||||
|
console.log(` ${r.id} — ${r.name}`);
|
||||||
|
}
|
||||||
|
console.log(`\n [DRY RUN] Would scan ${n(total)} skills`);
|
||||||
|
await client.end().catch(() => {});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process in batches
|
||||||
|
let processed = 0;
|
||||||
|
let passCount = 0;
|
||||||
|
let warnCount = 0;
|
||||||
|
let failCount = 0;
|
||||||
|
let errorCount = 0;
|
||||||
|
let totalScore = 0;
|
||||||
|
|
||||||
|
while (processed < total) {
|
||||||
|
const batch = await query(`
|
||||||
|
SELECT id, raw_content, cached_files
|
||||||
|
FROM skills
|
||||||
|
WHERE is_blocked = false
|
||||||
|
AND is_duplicate = false
|
||||||
|
AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection'))
|
||||||
|
AND security_status IS NULL
|
||||||
|
AND raw_content IS NOT NULL
|
||||||
|
ORDER BY github_stars DESC NULLS LAST
|
||||||
|
LIMIT $1
|
||||||
|
`, [BATCH_SIZE]);
|
||||||
|
|
||||||
|
if (batch.rows.length === 0) break;
|
||||||
|
|
||||||
|
for (const row of batch.rows) {
|
||||||
|
try {
|
||||||
|
const scripts = extractScripts(row.cached_files);
|
||||||
|
const report = scanSecurity({ content: row.raw_content, scripts });
|
||||||
|
|
||||||
|
await query(`
|
||||||
|
UPDATE skills
|
||||||
|
SET security_score = $1,
|
||||||
|
security_status = $2,
|
||||||
|
last_scanned = NOW(),
|
||||||
|
review_status = CASE
|
||||||
|
WHEN review_status IS NULL OR review_status = 'unreviewed'
|
||||||
|
THEN 'auto-scored'
|
||||||
|
ELSE review_status
|
||||||
|
END
|
||||||
|
WHERE id = $3
|
||||||
|
`, [report.score, report.status, row.id]);
|
||||||
|
|
||||||
|
totalScore += report.score;
|
||||||
|
if (report.status === 'pass') passCount++;
|
||||||
|
else if (report.status === 'warning') warnCount++;
|
||||||
|
else failCount++;
|
||||||
|
} catch (err) {
|
||||||
|
errorCount++;
|
||||||
|
if (errorCount <= 5) {
|
||||||
|
console.error(`\n Error scanning ${row.id}: ${err.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
processed++;
|
||||||
|
if (processed % 100 === 0 || processed === total) {
|
||||||
|
progress(processed, total);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('\n'); // Clear progress line
|
||||||
|
|
||||||
|
// Summary
|
||||||
|
const avgScore = processed > 0 ? (totalScore / (processed - errorCount)).toFixed(1) : 0;
|
||||||
|
console.log(`
|
||||||
|
┌─────────────────────────────────────────────────────┐
|
||||||
|
│ SECURITY SCAN SUMMARY │
|
||||||
|
├─────────────────────────────────────────────────────┤
|
||||||
|
│ Total scanned: ${n(processed).padStart(8)} │
|
||||||
|
│ Errors (skipped): ${n(errorCount).padStart(8)} │
|
||||||
|
├─────────────────────────────────────────────────────┤
|
||||||
|
│ PASS: ${n(passCount).padStart(8)} │
|
||||||
|
│ WARNING: ${n(warnCount).padStart(8)} │
|
||||||
|
│ FAIL: ${n(failCount).padStart(8)} │
|
||||||
|
├─────────────────────────────────────────────────────┤
|
||||||
|
│ Avg security score:${avgScore.toString().padStart(8)} │
|
||||||
|
└─────────────────────────────────────────────────────┘
|
||||||
|
`);
|
||||||
|
|
||||||
|
const dur = ((Date.now() - t0) / 1000).toFixed(1);
|
||||||
|
console.log(`Completed in ${dur}s`);
|
||||||
|
|
||||||
|
await client.end().catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch(async e => {
|
||||||
|
console.error('FAILED:', e.message || e);
|
||||||
|
await client?.end().catch(() => {});
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
@@ -51,6 +51,9 @@ CREATE TABLE IF NOT EXISTS skills (
|
|||||||
is_blocked BOOLEAN DEFAULT FALSE, -- Blocked from re-indexing (owner requested removal)
|
is_blocked BOOLEAN DEFAULT FALSE, -- Blocked from re-indexing (owner requested removal)
|
||||||
last_scanned TIMESTAMP WITH TIME ZONE,
|
last_scanned TIMESTAMP WITH TIME ZONE,
|
||||||
|
|
||||||
|
-- Review pipeline
|
||||||
|
review_status TEXT DEFAULT 'unreviewed', -- 'unreviewed', 'auto-scored', 'ai-reviewed', 'verified', 'needs-re-review'
|
||||||
|
|
||||||
-- Content
|
-- Content
|
||||||
content_hash TEXT,
|
content_hash TEXT,
|
||||||
raw_content TEXT,
|
raw_content TEXT,
|
||||||
|
|||||||
@@ -44,6 +44,11 @@ RUN adduser --system --uid 1001 indexer
|
|||||||
# Copy only the bundled dist (no node_modules needed - everything is bundled)
|
# Copy only the bundled dist (no node_modules needed - everything is bundled)
|
||||||
COPY --from=builder --chown=indexer:nodejs /app/services/indexer/dist ./dist
|
COPY --from=builder --chown=indexer:nodejs /app/services/indexer/dist ./dist
|
||||||
|
|
||||||
|
# Curation scripts + skillhub-core (for batch-security, batch-score, curate)
|
||||||
|
COPY --from=builder --chown=indexer:nodejs /app/scripts/curation ./scripts/curation
|
||||||
|
COPY --from=builder --chown=indexer:nodejs /app/packages/core/dist ./packages/core/dist
|
||||||
|
RUN npm install --no-save pg gray-matter yaml
|
||||||
|
|
||||||
USER indexer
|
USER indexer
|
||||||
|
|
||||||
CMD ["node", "dist/worker.js"]
|
CMD ["node", "dist/worker.js"]
|
||||||
|
|||||||
Reference in New Issue
Block a user