sync: SEO canonical URLs, curation improvements, batch scripts, stats fix

- Dynamic canonical URLs and alternates for all pages (SEO fix)
- Shared seo.ts utility for canonical path generation
- Improved duplicate detection: standalone > aggregator priority, forks/created_at tie-breakers
- browseReadyFilter now includes unique aggregator skills
- Stats aligned with browseReadyFilter (was showing 16k instead of 62k+)
- Re-review mechanism: content_hash changes trigger needs-re-review
- review_status field added to skills schema
- Batch security scan and quality score scripts
- Indexer Dockerfile updated for curation deps
- Removed roadmap from README

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
airano
2026-02-21 23:40:03 +03:30
parent 77917bf91f
commit 3f599a23fb
30 changed files with 1092 additions and 50 deletions

View File

@@ -184,18 +184,6 @@ Always review skill source code before installing.
---
## Roadmap
- [x] Web marketplace with growing skill catalog
- [x] CLI tool
- [x] Multi-platform support
- [x] Security scanning
- [ ] Claude Code MCP plugin
- [ ] AI-powered recommendations
- [ ] Enterprise features
---
## License
MIT - See [LICENSE](./LICENSE) for details.

View File

@@ -2,6 +2,20 @@ import { getTranslations, setRequestLocale } from 'next-intl/server';
import { Header } from '@/components/Header';
import { Footer } from '@/components/Footer';
import { Code2, Globe, Shield } from 'lucide-react';
import { getPageAlternates } from '@/lib/seo';
export async function generateMetadata({
params,
}: {
params: Promise<{ locale: string }>;
}) {
const { locale } = await params;
return {
alternates: getPageAlternates(locale, '/about'),
};
}
export default async function AboutPage({
params,

View File

@@ -3,6 +3,8 @@ import Link from 'next/link';
import { Header } from '@/components/Header';
import { Footer } from '@/components/Footer';
import { Github, Heart, Users, Code, GitFork, ExternalLink, Database, Clock } from 'lucide-react';
import { getPageAlternates } from '@/lib/seo';
export const dynamic = 'force-dynamic';
@@ -65,6 +67,18 @@ function formatLicenseName(license: string, locale: string): string {
return names ? names[locale as 'en' | 'fa'] || names.en : license;
}
export async function generateMetadata({
params,
}: {
params: Promise<{ locale: string }>;
}) {
const { locale } = await params;
return {
alternates: getPageAlternates(locale, '/attribution'),
};
}
export default async function AttributionPage({
params,
}: {

View File

@@ -5,6 +5,8 @@ import { createDb, skillQueries, categoryQueries } from '@skillhub/db';
import { BrowseFilters, SearchBar, Pagination, ActiveFilters, EmptyState } from '@/components/BrowseFilters';
import { SkillCard } from '@/components/SkillCard';
import { toPersianNumber } from '@/lib/format-number';
import { getPageAlternates } from '@/lib/seo';
// Force dynamic rendering to fetch fresh data from database
export const dynamic = 'force-dynamic';
@@ -79,6 +81,18 @@ async function getSkills(params: {
}
}
export async function generateMetadata({
params,
}: {
params: Promise<{ locale: string }>;
}) {
const { locale } = await params;
return {
alternates: getPageAlternates(locale, '/browse'),
};
}
export default async function BrowsePage({ params, searchParams }: BrowsePageProps) {
const { locale } = await params;
setRequestLocale(locale);

View File

@@ -31,6 +31,8 @@ import {
} from 'lucide-react';
import { createDb, categoryQueries } from '@skillhub/db';
import { formatNumber } from '@/lib/format-number';
import { getPageAlternates } from '@/lib/seo';
// Force dynamic rendering to fetch fresh data from database
export const dynamic = 'force-dynamic';
@@ -96,6 +98,18 @@ async function getHierarchicalCategories() {
}
}
export async function generateMetadata({
params,
}: {
params: Promise<{ locale: string }>;
}) {
const { locale } = await params;
return {
alternates: getPageAlternates(locale, '/categories'),
};
}
export default async function CategoriesPage({
params,
}: {

View File

@@ -2,9 +2,23 @@ import { getTranslations, setRequestLocale } from 'next-intl/server';
import { Header } from '@/components/Header';
import { Footer } from '@/components/Footer';
import { ClaimForm } from '@/components/ClaimForm';
import { getPageAlternates } from '@/lib/seo';
export const dynamic = 'force-dynamic';
export async function generateMetadata({
params,
}: {
params: Promise<{ locale: string }>;
}) {
const { locale } = await params;
return {
alternates: getPageAlternates(locale, '/claim'),
};
}
export default async function ClaimPage({
params,
}: {

View File

@@ -1,4 +1,5 @@
import { getTranslations, setRequestLocale } from 'next-intl/server';
import { getPageAlternates } from '@/lib/seo';
import Link from 'next/link';
import type { Metadata } from 'next';
import {
@@ -31,6 +32,7 @@ export async function generateMetadata({
return {
title: t('metadata.title'),
description: t('metadata.description'),
alternates: getPageAlternates(locale, '/claude-plugin'),
openGraph: {
title: t('metadata.title'),
description: t('metadata.description'),

View File

@@ -1,4 +1,18 @@
import { redirect } from 'next/navigation';
import { getPageAlternates } from '@/lib/seo';
export async function generateMetadata({
params,
}: {
params: Promise<{ locale: string }>;
}) {
const { locale } = await params;
return {
alternates: getPageAlternates(locale, '/contact'),
};
}
export default async function ContactPage({
params,

View File

@@ -5,6 +5,20 @@ import { ApiEndpointSection } from '@/components/ApiEndpointSection';
import type { EndpointDef } from '@/components/ApiEndpointSection';
import Link from 'next/link';
import { ArrowLeft, ArrowRight, Search, FileCode, Users, Compass, Mail } from 'lucide-react';
import { getPageAlternates } from '@/lib/seo';
export async function generateMetadata({
params,
}: {
params: Promise<{ locale: string }>;
}) {
const { locale } = await params;
return {
alternates: getPageAlternates(locale, '/docs/api'),
};
}
export default async function ApiDocsPage({
params,

View File

@@ -3,6 +3,20 @@ import { Header } from '@/components/Header';
import { Footer } from '@/components/Footer';
import Link from 'next/link';
import { ArrowLeft, ArrowRight } from 'lucide-react';
import { getPageAlternates } from '@/lib/seo';
export async function generateMetadata({
params,
}: {
params: Promise<{ locale: string }>;
}) {
const { locale } = await params;
return {
alternates: getPageAlternates(locale, '/docs/cli'),
};
}
export default async function CliDocsPage({
params,

View File

@@ -6,6 +6,8 @@ import Link from 'next/link';
import { ArrowLeft, ArrowRight } from 'lucide-react';
import { createDb, skills, sql } from '@skillhub/db';
import { formatPromptSkillCount } from '@/lib/format-number';
import { getPageAlternates } from '@/lib/seo';
export const dynamic = 'force-dynamic';
@@ -15,7 +17,7 @@ async function getSkillCount(): Promise<string> {
const result = await db
.select({ count: sql<number>`count(*)::int` })
.from(skills)
.where(sql`${skills.isDuplicate} = false AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`);
.where(sql`${skills.isDuplicate} = false`);
return formatPromptSkillCount(result[0]?.count ?? 16000);
} catch {
return '16,000+';
@@ -56,6 +58,18 @@ Search for unfamiliar tech or complex tasks. Only install "Pass" security status
};
}
export async function generateMetadata({
params,
}: {
params: Promise<{ locale: string }>;
}) {
const { locale } = await params;
return {
alternates: getPageAlternates(locale, '/docs/getting-started'),
};
}
export default async function GettingStartedPage({
params,
}: {

View File

@@ -3,6 +3,20 @@ import { Header } from '@/components/Header';
import { Footer } from '@/components/Footer';
import { BookOpen, Terminal, Code } from 'lucide-react';
import Link from 'next/link';
import { getPageAlternates } from '@/lib/seo';
export async function generateMetadata({
params,
}: {
params: Promise<{ locale: string }>;
}) {
const { locale } = await params;
return {
alternates: getPageAlternates(locale, '/docs'),
};
}
export default async function DocsPage({
params,

View File

@@ -6,6 +6,8 @@ import { Footer } from '@/components/Footer';
import { FavoritesList } from '@/components/FavoritesList';
import { FavoritesSignIn } from '@/components/FavoritesSignIn';
import { createDb, userQueries } from '@skillhub/db';
import { getPageAlternates } from '@/lib/seo';
// Force dynamic rendering
export const dynamic = 'force-dynamic';
@@ -14,6 +16,18 @@ interface FavoritesPageProps {
params: Promise<{ locale: string }>;
}
export async function generateMetadata({
params,
}: {
params: Promise<{ locale: string }>;
}) {
const { locale } = await params;
return {
alternates: getPageAlternates(locale, '/favorites'),
};
}
export default async function FavoritesPage({ params }: FavoritesPageProps) {
const { locale } = await params;
setRequestLocale(locale);

View File

@@ -5,6 +5,8 @@ import { createDb, skillQueries } from '@skillhub/db';
import { toPersianNumber } from '@/lib/format-number';
import { Pagination } from '@/components/BrowseFilters';
import { SkillCard } from '@/components/SkillCard';
import { getPageAlternates } from '@/lib/seo';
// Force dynamic rendering to fetch fresh data from database
export const dynamic = 'force-dynamic';
@@ -37,6 +39,18 @@ async function getFeaturedSkills(page: number, limit: number) {
}
}
export async function generateMetadata({
params,
}: {
params: Promise<{ locale: string }>;
}) {
const { locale } = await params;
return {
alternates: getPageAlternates(locale, '/featured'),
};
}
export default async function FeaturedPage({
params,
searchParams,

View File

@@ -7,6 +7,8 @@ import { Clock, RefreshCw } from 'lucide-react';
import { createDb, skillQueries } from '@skillhub/db';
import { toPersianNumber } from '@/lib/format-number';
import { Pagination } from '@/components/BrowseFilters';
import { getPageAlternates } from '@/lib/seo';
// Force dynamic rendering to fetch fresh data from database
export const dynamic = 'force-dynamic';
@@ -75,6 +77,18 @@ async function getTabCounts() {
}
}
export async function generateMetadata({
params,
}: {
params: Promise<{ locale: string }>;
}) {
const { locale } = await params;
return {
alternates: getPageAlternates(locale, '/new'),
};
}
export default async function NewSkillsPage({
params,
searchParams,

View File

@@ -7,9 +7,23 @@ import { createDb, skillQueries } from '@skillhub/db';
import { formatCompactNumber, toPersianNumber } from '@/lib/format-number';
import { ExternalLink, Download, Package, Eye, GitFork, ArrowUpDown, FolderGit2 } from 'lucide-react';
import Link from 'next/link';
import type { Metadata } from 'next';
import { getPageAlternates } from '@/lib/seo';
export const dynamic = 'force-dynamic';
export async function generateMetadata({
params,
}: {
params: Promise<{ locale: string; username: string }>;
}): Promise<Metadata> {
const { locale, username } = await params;
return {
title: `${decodeURIComponent(username)} | SkillHub`,
alternates: getPageAlternates(locale, `/owner/${username}`),
};
}
const ITEMS_PER_PAGE = 24;
type SortOption = 'popularity' | 'downloads' | 'stars';
@@ -183,8 +197,7 @@ export default async function OwnerPage({ params, searchParams }: OwnerPageProps
<span className="text-sm text-text-secondary">{t('repo.filter')}:</span>
<Link
href={buildUrl({ repo: '', page: 1 })}
className={`px-3 py-1 text-xs rounded-full transition-colors ${
!activeRepo
className={`px-3 py-1 text-xs rounded-full transition-colors ${!activeRepo
? 'bg-primary text-primary-foreground font-medium'
: 'bg-surface-elevated text-text-secondary hover:bg-surface-subtle border border-border'
}`}
@@ -195,8 +208,7 @@ export default async function OwnerPage({ params, searchParams }: OwnerPageProps
<Link
key={r.repo}
href={buildUrl({ repo: r.repo, page: 1 })}
className={`px-3 py-1 text-xs rounded-full transition-colors ${
activeRepo === r.repo
className={`px-3 py-1 text-xs rounded-full transition-colors ${activeRepo === r.repo
? 'bg-primary text-primary-foreground font-medium'
: 'bg-surface-elevated text-text-secondary hover:bg-surface-subtle border border-border'
}`}
@@ -227,8 +239,7 @@ export default async function OwnerPage({ params, searchParams }: OwnerPageProps
<Link
key={opt.value}
href={buildUrl({ sort: opt.value, page: 1 })}
className={`px-3 py-1.5 text-sm rounded-lg transition-colors ${
sort === opt.value
className={`px-3 py-1.5 text-sm rounded-lg transition-colors ${sort === opt.value
? 'bg-primary text-primary-foreground font-medium'
: 'bg-surface-elevated text-text-secondary hover:bg-surface-subtle border border-border'
}`}

View File

@@ -7,6 +7,8 @@ import { HeroSearch } from '@/components/HeroSearch';
import { SkillCard } from '@/components/SkillCard';
import { createDb, categoryQueries, skillQueries, skills, sql } from '@skillhub/db';
import { formatCompactNumber } from '@/lib/format-number';
import { getPageAlternates } from '@/lib/seo';
// Force dynamic rendering to fetch fresh data from database
export const dynamic = 'force-dynamic';
@@ -16,8 +18,8 @@ async function getStats() {
try {
const db = createDb();
// Browse-ready filter: exclude duplicates and aggregators
const browseReady = sql`${skills.isDuplicate} = false AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`;
// Browse-ready filter: exclude duplicates (matches browseReadyFilter in queries.ts)
const browseReady = sql`${skills.isDuplicate} = false`;
// Get total skills count (browse-ready, SKILL.md only)
const skillsResult = await db
@@ -82,6 +84,18 @@ async function getFeaturedSkills() {
}
export async function generateMetadata({
params,
}: {
params: Promise<{ locale: string }>;
}) {
const { locale } = await params;
return {
alternates: getPageAlternates(locale, '/'),
};
}
export default async function HomePage({
params,
}: {

View File

@@ -3,6 +3,8 @@ import Link from 'next/link';
import { Header } from '@/components/Header';
import { Footer } from '@/components/Footer';
import { Shield, Database, Cookie, Users, Lock, Mail, ArrowRight, ArrowLeft } from 'lucide-react';
import { getPageAlternates } from '@/lib/seo';
export const dynamic = 'force-dynamic';
@@ -15,6 +17,18 @@ const sectionIcons = {
contact: Mail,
};
export async function generateMetadata({
params,
}: {
params: Promise<{ locale: string }>;
}) {
const { locale } = await params;
return {
alternates: getPageAlternates(locale, '/privacy'),
};
}
export default async function PrivacyPage({
params,
}: {

View File

@@ -16,10 +16,32 @@ import { createDb, skillQueries } from '@skillhub/db';
import { FORMAT_LABELS } from 'skillhub-core';
import { formatCompactNumber } from '@/lib/format-number';
import { shouldCountView } from '@/lib/cache';
import type { Metadata } from 'next';
import { getPageAlternates } from '@/lib/seo';
// Force dynamic rendering to fetch fresh data from database
export const dynamic = 'force-dynamic';
export async function generateMetadata({
params,
}: {
params: Promise<{ locale: string; id: string[] }>;
}): Promise<Metadata> {
const { locale, id } = await params;
const skillId = id.join('/');
// Optionally fetch skill name for title
const dbSkill = await getSkill(skillId);
const title = dbSkill ? `${dbSkill.name} | SkillHub` : 'SkillHub';
const description = dbSkill ? dbSkill.description : undefined;
return {
title,
description,
alternates: getPageAlternates(locale, `/skill/${skillId}`),
};
}
interface SkillPageProps {
params: Promise<{ locale: string; id: string[] }>;
}
@@ -71,9 +93,9 @@ export default async function SkillPage({ params }: SkillPageProps) {
const db = createDb();
shouldCountView(dbSkill.id, clientIp).then((shouldCount) => {
if (shouldCount) {
skillQueries.incrementViews(db, dbSkill.id).catch(() => {});
skillQueries.incrementViews(db, dbSkill.id).catch(() => { });
}
}).catch(() => {});
}).catch(() => { });
}
// Map database response to expected format

View File

@@ -2,6 +2,8 @@ import { getTranslations, setRequestLocale } from 'next-intl/server';
import { Header } from '@/components/Header';
import { Footer } from '@/components/Footer';
import { Mail, Bitcoin, ExternalLink } from 'lucide-react';
import { getPageAlternates } from '@/lib/seo';
export const dynamic = 'force-dynamic';
@@ -9,6 +11,18 @@ interface SupportPageProps {
params: Promise<{ locale: string }>;
}
export async function generateMetadata({
params,
}: {
params: Promise<{ locale: string }>;
}) {
const { locale } = await params;
return {
alternates: getPageAlternates(locale, '/support'),
};
}
export default async function SupportPage({ params }: SupportPageProps) {
const { locale } = await params;
setRequestLocale(locale);

View File

@@ -3,6 +3,8 @@ import Link from 'next/link';
import { Header } from '@/components/Header';
import { Footer } from '@/components/Footer';
import { FileText, UserCheck, AlertTriangle, Scale, Trash2, RefreshCw, ArrowRight, ArrowLeft } from 'lucide-react';
import { getPageAlternates } from '@/lib/seo';
export const dynamic = 'force-dynamic';
@@ -15,6 +17,18 @@ const sectionIcons = {
changes: RefreshCw,
};
export async function generateMetadata({
params,
}: {
params: Promise<{ locale: string }>;
}) {
const { locale } = await params;
return {
alternates: getPageAlternates(locale, '/terms'),
};
}
export default async function TermsPage({
params,
}: {

View File

@@ -34,8 +34,8 @@ export async function GET(request: NextRequest) {
});
}
// Browse-ready filter: exclude duplicates and aggregators
const browseReady = sql`${skills.isDuplicate} = false AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`;
// Browse-ready filter: exclude duplicates (matches browseReadyFilter in queries.ts)
const browseReady = sql`${skills.isDuplicate} = false`;
// Browse-ready skill stats (skills count + contributors)
const statsResult = await db

View File

@@ -1,17 +1,8 @@
import type { MetadataRoute } from 'next';
import { createDb, skillQueries, categoryQueries } from '@skillhub/db';
const BASE_URL =
process.env.NEXT_PUBLIC_APP_URL || 'https://skills.palebluedot.live';
import { getCanonicalPath, primaryDomain as BASE_URL } from '@/lib/seo';
const locales = ['en', 'fa'] as const;
const DEFAULT_LOCALE = 'en';
/** Get canonical URL path for a locale (no prefix for default locale, matching localePrefix: 'as-needed') */
function canonicalPath(locale: string, route: string): string {
if (locale === DEFAULT_LOCALE) return route || '/';
return `/${locale}${route}`;
}
const staticRoutes = [
'',
@@ -43,13 +34,13 @@ function makeEntry(
}
): MetadataRoute.Sitemap[number] {
return {
url: `${BASE_URL}${canonicalPath(locale, route)}`,
url: `${BASE_URL}${getCanonicalPath(locale, route)}`,
lastModified: options?.lastModified,
changeFrequency: options?.changeFrequency,
priority: options?.priority,
alternates: {
languages: Object.fromEntries(
locales.map((l) => [l, `${BASE_URL}${canonicalPath(l, route)}`])
locales.map((l) => [l, `${BASE_URL}${getCanonicalPath(l, route)}`])
),
},
};

50
apps/web/lib/seo.ts Normal file
View File

@@ -0,0 +1,50 @@
import { locales } from '@/i18n';
export const primaryDomain =
process.env.NEXT_PUBLIC_APP_URL || 'https://skills.palebluedot.live';
const DEFAULT_LOCALE = 'en';
/**
* Get canonical URL path for a locale
* (no prefix for default locale, matching localePrefix: 'as-needed')
*/
export function getCanonicalPath(locale: string, route: string): string {
// Normalize route to always start with a slash and not end with a slash
let formattedRoute = route;
if (!formattedRoute.startsWith('/')) {
formattedRoute = `/${formattedRoute}`;
}
if (formattedRoute.length > 1 && formattedRoute.endsWith('/')) {
formattedRoute = formattedRoute.slice(0, -1);
}
if (locale === DEFAULT_LOCALE) {
if (formattedRoute === '/') return '';
return formattedRoute;
}
if (formattedRoute === '/') return `/${locale}`;
return `/${locale}${formattedRoute}`;
}
/**
* Generate Next.js alternates metadata for a specific route
* @param locale Current active locale
* @param route The route path (e.g. '/about', '/categories')
*/
export function getPageAlternates(locale: string, route: string) {
// Ensure we fall back to / if it's empty
const canonicalPath = getCanonicalPath(locale, route) || '/';
const languages: Record<string, string> = {};
for (const l of locales) {
languages[l] = `${primaryDomain}${getCanonicalPath(l, route) || '/'}`;
}
return {
canonical: `${primaryDomain}${canonicalPath}`,
languages,
};
}

View File

@@ -31,11 +31,11 @@ function getRawClientLocal() {
type DB = PostgresJsDatabase<typeof schema>;
/**
* Curation filter: excludes duplicates and aggregator-only skills.
* Curation filter: excludes duplicates.
* Skills with skill_type=NULL (newly crawled, not yet curated) are included
* so they don't disappear between curate.mjs runs.
*/
const browseReadyFilter = sql`(${skills.isDuplicate} = false) AND (${skills.skillType} IS NULL OR ${skills.skillType} != 'aggregator')`;
const browseReadyFilter = sql`(${skills.isDuplicate} = false)`;
/**
* Skill queries
@@ -556,17 +556,38 @@ export const skillQueries = {
skill: typeof skills.$inferInsert
): Promise<typeof skills.$inferSelect> => {
// Check if commitSha changed (for cache invalidation)
// and if content_hash changed (for re-review mechanism)
let shouldClearCache = false;
if (skill.id && skill.commitSha) {
let shouldTriggerReReview = false;
if (skill.id) {
const existing = await db
.select({ commitSha: skills.commitSha, cachedFiles: skills.cachedFiles })
.select({
commitSha: skills.commitSha,
cachedFiles: skills.cachedFiles,
contentHash: skills.contentHash,
reviewStatus: skills.reviewStatus,
})
.from(skills)
.where(eq(skills.id, skill.id))
.limit(1);
if (existing[0] && existing[0].cachedFiles && existing[0].commitSha !== skill.commitSha) {
if (existing[0]) {
// Cache invalidation: clear cachedFiles when commitSha changes
if (skill.commitSha && existing[0].cachedFiles && existing[0].commitSha !== skill.commitSha) {
shouldClearCache = true;
}
// Re-review mechanism (Phase 6.4): when content_hash changes and skill
// was previously reviewed, mark it for re-review
if (
skill.contentHash &&
existing[0].contentHash &&
existing[0].contentHash !== skill.contentHash &&
(existing[0].reviewStatus === 'verified' || existing[0].reviewStatus === 'ai-reviewed')
) {
shouldTriggerReReview = true;
}
}
}
const result = await db
@@ -605,6 +626,9 @@ export const skillQueries = {
// which only updates it when content-related columns actually change
// Clear cachedFiles if commitSha changed
...(shouldClearCache ? { cachedFiles: null } : {}),
// Re-review mechanism (Phase 6.4): mark previously reviewed skills for re-review
// when their content changes. Only affects 'verified' and 'ai-reviewed' skills.
...(shouldTriggerReReview ? { reviewStatus: 'needs-re-review' } : {}),
},
})
.returning();
@@ -1172,7 +1196,7 @@ export async function runPostCrawlCuration(db: DB): Promise<{ classified: number
`);
results.classified = (classResult as unknown as { rowCount: number }).rowCount ?? 0;
// Step 4: Mark duplicates by content_hash (keep canonical = most stars)
// Step 4: Mark duplicates by content_hash (keep canonical = most stars, prioritizing standalone over aggregator)
const dupResult = await db.execute(sql`
UPDATE skills s SET is_duplicate = true
WHERE s.is_blocked = false
@@ -1183,8 +1207,17 @@ export async function runPostCrawlCuration(db: DB): Promise<{ classified: number
WHERE s2.content_hash = s.content_hash
AND s2.is_blocked = false
AND s2.id != s.id
AND (s2.github_stars > s.github_stars
OR (s2.github_stars = s.github_stars AND s2.indexed_at < s.indexed_at))
AND (
(s2.skill_type != 'aggregator' AND s.skill_type = 'aggregator')
OR (
(s2.skill_type = 'aggregator') = (s.skill_type = 'aggregator')
AND (
s2.github_stars > s.github_stars
OR (s2.github_stars = s.github_stars AND s2.github_forks > s.github_forks)
OR (s2.github_stars = s.github_stars AND s2.github_forks = s.github_forks AND s2.created_at < s.created_at)
)
)
)
)
`);
results.duplicates = (dupResult as unknown as { rowCount: number }).rowCount ?? 0;
@@ -1198,7 +1231,6 @@ export async function runPostCrawlCuration(db: DB): Promise<{ classified: number
JOIN skills s ON sc.skill_id = s.id
WHERE s.is_blocked = false
AND s.is_duplicate = false
AND (s.skill_type IS NULL OR s.skill_type != 'aggregator')
GROUP BY sc.category_id
) sub
WHERE c.id = sub.category_id

View File

@@ -79,6 +79,7 @@ export const skills = pgTable(
isDuplicate: boolean('is_duplicate').default(false),
canonicalSkillId: text('canonical_skill_id'), // points to the "original" if this is a duplicate
repoSkillCount: integer('repo_skill_count'), // cached count of skills in same repo
reviewStatus: text('review_status').default('unreviewed'), // unreviewed → auto-scored → ai-reviewed → verified (or needs-re-review)
// Content (cached)
contentHash: text('content_hash'),

View File

@@ -0,0 +1,459 @@
#!/usr/bin/env node
/**
* Phase 5: Batch Quality Score
*
* Calculates quality scores for browse-ready skills using the same logic
* as SkillAnalyzer.calculateQuality() from services/indexer/src/analyzer.ts.
*
* Updates: quality_score, quality_details, review_status → 'auto-scored'
*
* Usage:
* DATABASE_URL=postgres://... node scripts/curation/batch-score.mjs
*
* Options:
* --dry-run Show what would change without writing
* --batch-size=N Process N skills per batch (default 100)
*/
import { createRequire } from 'module';
import { resolve, dirname } from 'path';
import { fileURLToPath, pathToFileURL } from 'url';
const __dirname = dirname(fileURLToPath(import.meta.url));
// ─── Find pg module ───
let pg;
const tryPaths = [
resolve(__dirname, '../..', 'package.json'),
'/tmp/package.json',
process.env.APPDATA ? resolve(process.env.APPDATA, '..', 'Local', 'Temp', 'package.json') : null,
].filter(Boolean);
for (const p of tryPaths) {
try { pg = createRequire(p)('pg'); break; } catch {}
}
if (!pg) { console.error('pg not found. Run: npm install pg'); process.exit(1); }
// ─── Find skillhub-core (ESM-only — use direct path import) ───
let parseSkillMd, parseGenericInstructionFile, validateSkill, scanSecurity;
const corePaths = [
resolve(__dirname, '../../packages/core/dist/index.js'),
resolve(__dirname, '../../node_modules/skillhub-core/dist/index.js'),
resolve(__dirname, '../../services/indexer/node_modules/skillhub-core/dist/index.js'),
];
for (const p of corePaths) {
try {
const core = await import(pathToFileURL(p).href);
parseSkillMd = core.parseSkillMd;
parseGenericInstructionFile = core.parseGenericInstructionFile;
validateSkill = core.validateSkill;
scanSecurity = core.scanSecurity;
break;
} catch {}
}
if (!parseSkillMd) { console.error('skillhub-core not found. Run: pnpm build'); process.exit(1); }
// ─── Config ───
const DATABASE_URL = process.env.DATABASE_URL || 'postgresql://postgres:postgres@localhost:5432/skillhub';
const DRY_RUN = process.argv.includes('--dry-run');
const batchArg = process.argv.find(a => a.startsWith('--batch-size='));
const BATCH_SIZE = batchArg ? parseInt(batchArg.split('=')[1]) : 100;
// ─── Helpers ───
const n = v => Number(v ?? 0).toLocaleString('en-US');
function progress(current, total) {
const pct = ((current / total) * 100).toFixed(1);
process.stdout.write(`\r Processing ${n(current)} / ${n(total)} (${pct}%)`);
}
// ─── Database ───
let client;
async function connect() {
client = new pg.Client({
connectionString: DATABASE_URL,
ssl: false,
connectionTimeoutMillis: 15000,
query_timeout: 600000,
keepAlive: true,
});
await client.connect();
console.log('Connected to database');
}
async function query(sql, params = []) {
return client.query(sql, params);
}
// ─── Quality scoring (mirrors SkillAnalyzer.calculateQuality) ───
function scoreDocumentation(skill, scripts, references) {
let score = 0;
// Has description (required)
if (skill.metadata.description && skill.metadata.description.length > 20) {
score += 20;
}
// Content length and structure
const contentLength = skill.content.length;
if (contentLength > 500) score += 15;
else if (contentLength > 200) score += 10;
else if (contentLength > 50) score += 5;
// Has headers (good structure)
const headerCount = (skill.content.match(/^#+\s/gm) || []).length;
if (headerCount >= 3) score += 15;
else if (headerCount >= 1) score += 10;
// Has code examples
if (skill.content.includes('```')) {
score += 15;
}
// Has version
if (skill.metadata.version) {
score += 10;
}
// Has license
if (skill.metadata.license) {
score += 5;
}
// Has compatibility info
if (skill.metadata.compatibility?.platforms?.length) {
score += 10;
}
// Has scripts
if (scripts.length > 0) {
score += 5;
}
// Has references
if (references.length > 0) {
score += 5;
}
return Math.min(100, score);
}
function scoreMaintenance(repoMeta) {
let score = 0;
// Check last update time
const lastUpdate = new Date(repoMeta.updatedAt);
const daysSinceUpdate = (Date.now() - lastUpdate.getTime()) / (1000 * 60 * 60 * 24);
if (daysSinceUpdate < 30) score += 40;
else if (daysSinceUpdate < 90) score += 30;
else if (daysSinceUpdate < 180) score += 20;
else if (daysSinceUpdate < 365) score += 10;
// Has license
if (repoMeta.license) {
score += 20;
}
// Has description
if (repoMeta.description) {
score += 10;
}
// Has topics (always [] since not in DB — will miss ~10 points)
if (repoMeta.topics.length > 0) {
score += 10;
}
// Activity level (forks indicate usage)
if (repoMeta.forks >= 10) score += 20;
else if (repoMeta.forks >= 5) score += 15;
else if (repoMeta.forks >= 1) score += 10;
return Math.min(100, score);
}
function scorePopularity(repoMeta) {
const stars = repoMeta.stars;
const forks = repoMeta.forks;
let score = 0;
if (stars >= 1000) score += 50;
else if (stars >= 100) score += 40;
else if (stars >= 50) score += 30;
else if (stars >= 10) score += 20;
else if (stars >= 5) score += 10;
else if (stars >= 1) score += 5;
if (forks >= 50) score += 30;
else if (forks >= 10) score += 20;
else if (forks >= 5) score += 15;
else if (forks >= 1) score += 10;
// Bonus for relevant topics (always [] from DB — will miss ~20 points)
const relevantTopics = ['ai', 'agent', 'skill', 'claude', 'copilot', 'codex', 'llm'];
const hasRelevantTopic = repoMeta.topics.some(t =>
relevantTopics.some(rt => t.toLowerCase().includes(rt))
);
if (hasRelevantTopic) {
score += 20;
}
return Math.min(100, score);
}
function calculateQuality(skill, repoMeta, securityScore, validation, scripts, references) {
const factors = [];
// Documentation quality (30% weight)
const docScore = scoreDocumentation(skill, scripts, references);
factors.push({ name: 'documentation', score: docScore, weight: 0.3 });
// Maintenance signals (25% weight)
const maintScore = scoreMaintenance(repoMeta);
factors.push({ name: 'maintenance', score: maintScore, weight: 0.25 });
// Popularity (20% weight)
const popScore = scorePopularity(repoMeta);
factors.push({ name: 'popularity', score: popScore, weight: 0.2 });
// Security (15% weight)
factors.push({ name: 'security', score: securityScore, weight: 0.15 });
// Validation (10% weight)
const valScore = validation.isValid ? 100 : Math.max(0, 100 - validation.errors.length * 20);
factors.push({ name: 'validation', score: valScore, weight: 0.1 });
// Calculate weighted overall score
const overall = Math.round(
factors.reduce((sum, f) => sum + f.score * f.weight, 0)
);
return {
overall,
documentation: docScore,
maintenance: maintScore,
popularity: popScore,
factors,
};
}
// ─── Extract scripts/references from cached_files ───
function extractScripts(cachedFiles) {
if (!cachedFiles || !cachedFiles.items) return [];
return cachedFiles.items
.filter(item =>
!item.isBinary &&
item.name !== 'SKILL.md' &&
(item.name.endsWith('.sh') ||
item.name.endsWith('.py') ||
item.name.endsWith('.js') ||
item.name.endsWith('.ts') ||
item.name.endsWith('.ps1') ||
item.name.endsWith('.bat') ||
item.name.endsWith('.rb'))
)
.map(item => ({ name: item.name, content: item.content }));
}
function extractReferences(cachedFiles) {
if (!cachedFiles || !cachedFiles.items) return [];
return cachedFiles.items
.filter(item =>
!item.isBinary &&
item.name !== 'SKILL.md' &&
(item.name.endsWith('.md') || item.name.endsWith('.txt'))
)
.map(item => ({ name: item.name, content: item.content }));
}
// ─── Main ───
async function main() {
const t0 = Date.now();
await connect();
console.log(`\n${'='.repeat(70)}`);
console.log(' BATCH QUALITY SCORE (Phase 5)');
console.log(`${'='.repeat(70)}`);
if (DRY_RUN) console.log('\n *** DRY RUN MODE — no changes will be made ***\n');
// Count skills to score
const countResult = await query(`
SELECT COUNT(*)::int AS total
FROM skills
WHERE is_blocked = false
AND is_duplicate = false
AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection'))
AND quality_score IS NULL
AND raw_content IS NOT NULL
`);
const total = countResult.rows[0].total;
console.log(` Skills to score: ${n(total)}`);
console.log(` Batch size: ${BATCH_SIZE}`);
if (total === 0) {
console.log(' Nothing to do — all browse-ready skills already scored');
await client.end().catch(() => {});
return;
}
if (DRY_RUN) {
const sample = await query(`
SELECT id, LEFT(name, 50) AS name, github_stars
FROM skills
WHERE is_blocked = false
AND is_duplicate = false
AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection'))
AND quality_score IS NULL
AND raw_content IS NOT NULL
ORDER BY github_stars DESC NULLS LAST
LIMIT 5
`);
console.log('\n Sample skills that would be scored:');
for (const r of sample.rows) {
console.log(` [${n(r.github_stars)}★] ${r.id}${r.name}`);
}
console.log(`\n [DRY RUN] Would score ${n(total)} skills`);
await client.end().catch(() => {});
return;
}
// Process in batches
let processed = 0;
let errorCount = 0;
let totalScore = 0;
const scoreBuckets = { excellent: 0, good: 0, fair: 0, poor: 0 };
while (processed < total) {
const batch = await query(`
SELECT id, raw_content, cached_files, source_format,
github_stars, github_forks, license, description,
updated_at, version, compatibility, security_score
FROM skills
WHERE is_blocked = false
AND is_duplicate = false
AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection'))
AND quality_score IS NULL
AND raw_content IS NOT NULL
ORDER BY github_stars DESC NULLS LAST
LIMIT $1
`, [BATCH_SIZE]);
if (batch.rows.length === 0) break;
for (const row of batch.rows) {
try {
// Parse the skill content
const sourceFormat = row.source_format || 'skill.md';
let skill;
if (sourceFormat === 'skill.md') {
skill = parseSkillMd(row.raw_content);
} else {
skill = parseGenericInstructionFile(row.raw_content, sourceFormat, {
name: row.description?.split(/\s+/).slice(0, 3).join('-') || 'skill',
description: row.description,
owner: '',
});
}
// Validate
const validation = sourceFormat === 'skill.md'
? validateSkill(skill)
: { isValid: true, errors: [], warnings: [] };
// Construct pseudo-repoMeta from DB fields
const repoMeta = {
stars: row.github_stars || 0,
forks: row.github_forks || 0,
license: row.license || null,
description: row.description || null,
updatedAt: row.updated_at ? row.updated_at.toISOString() : new Date().toISOString(),
defaultBranch: 'main',
topics: [], // NOT in DB — popularity factor will miss ~20 points
};
// Use already-computed security score (from Phase 4), default to 100 if not scanned
const securityScore = row.security_score ?? 100;
// Extract scripts/references from cached_files
const scripts = extractScripts(row.cached_files);
const references = extractReferences(row.cached_files);
// Calculate quality
const quality = calculateQuality(skill, repoMeta, securityScore, validation, scripts, references);
// Build quality_details for JSONB storage
const qualityDetails = {
documentation: quality.documentation,
maintenance: quality.maintenance,
popularity: quality.popularity,
factors: quality.factors,
};
await query(`
UPDATE skills
SET quality_score = $1,
quality_details = $2,
review_status = CASE
WHEN review_status IS NULL OR review_status = 'unreviewed'
THEN 'auto-scored'
ELSE review_status
END
WHERE id = $3
`, [quality.overall, JSON.stringify(qualityDetails), row.id]);
totalScore += quality.overall;
if (quality.overall >= 70) scoreBuckets.excellent++;
else if (quality.overall >= 50) scoreBuckets.good++;
else if (quality.overall >= 30) scoreBuckets.fair++;
else scoreBuckets.poor++;
} catch (err) {
errorCount++;
if (errorCount <= 5) {
console.error(`\n Error scoring ${row.id}: ${err.message}`);
}
}
processed++;
if (processed % 100 === 0 || processed === total) {
progress(processed, total);
}
}
}
console.log('\n'); // Clear progress line
// Summary
const scored = processed - errorCount;
const avgScore = scored > 0 ? (totalScore / scored).toFixed(1) : 0;
console.log(`
┌─────────────────────────────────────────────────────┐
│ QUALITY SCORE SUMMARY │
├─────────────────────────────────────────────────────┤
│ Total scored: ${n(scored).padStart(8)}
│ Errors (skipped): ${n(errorCount).padStart(8)}
├─────────────────────────────────────────────────────┤
│ Excellent (70+): ${n(scoreBuckets.excellent).padStart(8)}
│ Good (50-69): ${n(scoreBuckets.good).padStart(8)}
│ Fair (30-49): ${n(scoreBuckets.fair).padStart(8)}
│ Poor (<30): ${n(scoreBuckets.poor).padStart(8)}
├─────────────────────────────────────────────────────┤
│ Avg quality score: ${avgScore.toString().padStart(8)}
└─────────────────────────────────────────────────────┘
`);
console.log(' Note: repoMeta.topics not in DB — popularity score misses ~20 points');
const dur = ((Date.now() - t0) / 1000).toFixed(1);
console.log(`\nCompleted in ${dur}s`);
await client.end().catch(() => {});
}
main().catch(async e => {
console.error('FAILED:', e.message || e);
await client?.end().catch(() => {});
process.exit(1);
});

View File

@@ -0,0 +1,240 @@
#!/usr/bin/env node
/**
* Phase 4: Batch Security Scan
*
* Scans browse-ready skills for security issues using scanSecurity() from skillhub-core.
* Updates: security_score, security_status, last_scanned, review_status → 'auto-scored'
*
* Usage:
* DATABASE_URL=postgres://... node scripts/curation/batch-security.mjs
*
* Options:
* --dry-run Show what would change without writing
* --batch-size=N Process N skills per batch (default 100)
*/
import { createRequire } from 'module';
import { resolve, dirname } from 'path';
import { fileURLToPath, pathToFileURL } from 'url';
const __dirname = dirname(fileURLToPath(import.meta.url));
// ─── Find pg module ───
let pg;
const tryPaths = [
resolve(__dirname, '../..', 'package.json'),
'/tmp/package.json',
process.env.APPDATA ? resolve(process.env.APPDATA, '..', 'Local', 'Temp', 'package.json') : null,
].filter(Boolean);
for (const p of tryPaths) {
try { pg = createRequire(p)('pg'); break; } catch {}
}
if (!pg) { console.error('pg not found. Run: npm install pg'); process.exit(1); }
// ─── Find skillhub-core (ESM-only — use direct path import) ───
let scanSecurity;
const corePaths = [
resolve(__dirname, '../../packages/core/dist/index.js'),
resolve(__dirname, '../../node_modules/skillhub-core/dist/index.js'),
resolve(__dirname, '../../services/indexer/node_modules/skillhub-core/dist/index.js'),
];
for (const p of corePaths) {
try {
const core = await import(pathToFileURL(p).href);
scanSecurity = core.scanSecurity;
break;
} catch {}
}
if (!scanSecurity) { console.error('skillhub-core not found. Run: pnpm build'); process.exit(1); }
// ─── Config ───
const DATABASE_URL = process.env.DATABASE_URL || 'postgresql://postgres:postgres@localhost:5432/skillhub';
const DRY_RUN = process.argv.includes('--dry-run');
const batchArg = process.argv.find(a => a.startsWith('--batch-size='));
const BATCH_SIZE = batchArg ? parseInt(batchArg.split('=')[1]) : 100;
// ─── Helpers ───
const n = v => Number(v ?? 0).toLocaleString('en-US');
function progress(current, total) {
const pct = ((current / total) * 100).toFixed(1);
process.stdout.write(`\r Processing ${n(current)} / ${n(total)} (${pct}%)`);
}
// ─── Database ───
let client;
async function connect() {
client = new pg.Client({
connectionString: DATABASE_URL,
ssl: false,
connectionTimeoutMillis: 15000,
query_timeout: 600000,
keepAlive: true,
});
await client.connect();
console.log('Connected to database');
}
async function query(sql, params = []) {
return client.query(sql, params);
}
// ─── Extract scripts from cached_files ───
function extractScripts(cachedFiles) {
if (!cachedFiles || !cachedFiles.items) return [];
return cachedFiles.items
.filter(item =>
!item.isBinary &&
item.name !== 'SKILL.md' &&
(item.name.endsWith('.sh') ||
item.name.endsWith('.py') ||
item.name.endsWith('.js') ||
item.name.endsWith('.ts') ||
item.name.endsWith('.ps1') ||
item.name.endsWith('.bat') ||
item.name.endsWith('.rb'))
)
.map(item => ({ name: item.name, content: item.content }));
}
// ─── Main ───
async function main() {
const t0 = Date.now();
await connect();
console.log(`\n${'='.repeat(70)}`);
console.log(' BATCH SECURITY SCAN (Phase 4)');
console.log(`${'='.repeat(70)}`);
if (DRY_RUN) console.log('\n *** DRY RUN MODE — no changes will be made ***\n');
// Count skills to scan
const countResult = await query(`
SELECT COUNT(*)::int AS total
FROM skills
WHERE is_blocked = false
AND is_duplicate = false
AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection'))
AND security_status IS NULL
AND raw_content IS NOT NULL
`);
const total = countResult.rows[0].total;
console.log(` Skills to scan: ${n(total)}`);
console.log(` Batch size: ${BATCH_SIZE}`);
if (total === 0) {
console.log(' Nothing to do — all browse-ready skills already scanned');
await client.end().catch(() => {});
return;
}
if (DRY_RUN) {
// Show sample
const sample = await query(`
SELECT id, LEFT(name, 50) AS name
FROM skills
WHERE is_blocked = false
AND is_duplicate = false
AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection'))
AND security_status IS NULL
AND raw_content IS NOT NULL
LIMIT 5
`);
console.log('\n Sample skills that would be scanned:');
for (const r of sample.rows) {
console.log(` ${r.id}${r.name}`);
}
console.log(`\n [DRY RUN] Would scan ${n(total)} skills`);
await client.end().catch(() => {});
return;
}
// Process in batches
let processed = 0;
let passCount = 0;
let warnCount = 0;
let failCount = 0;
let errorCount = 0;
let totalScore = 0;
while (processed < total) {
const batch = await query(`
SELECT id, raw_content, cached_files
FROM skills
WHERE is_blocked = false
AND is_duplicate = false
AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection'))
AND security_status IS NULL
AND raw_content IS NOT NULL
ORDER BY github_stars DESC NULLS LAST
LIMIT $1
`, [BATCH_SIZE]);
if (batch.rows.length === 0) break;
for (const row of batch.rows) {
try {
const scripts = extractScripts(row.cached_files);
const report = scanSecurity({ content: row.raw_content, scripts });
await query(`
UPDATE skills
SET security_score = $1,
security_status = $2,
last_scanned = NOW(),
review_status = CASE
WHEN review_status IS NULL OR review_status = 'unreviewed'
THEN 'auto-scored'
ELSE review_status
END
WHERE id = $3
`, [report.score, report.status, row.id]);
totalScore += report.score;
if (report.status === 'pass') passCount++;
else if (report.status === 'warning') warnCount++;
else failCount++;
} catch (err) {
errorCount++;
if (errorCount <= 5) {
console.error(`\n Error scanning ${row.id}: ${err.message}`);
}
}
processed++;
if (processed % 100 === 0 || processed === total) {
progress(processed, total);
}
}
}
console.log('\n'); // Clear progress line
// Summary
const avgScore = processed > 0 ? (totalScore / (processed - errorCount)).toFixed(1) : 0;
console.log(`
┌─────────────────────────────────────────────────────┐
│ SECURITY SCAN SUMMARY │
├─────────────────────────────────────────────────────┤
│ Total scanned: ${n(processed).padStart(8)}
│ Errors (skipped): ${n(errorCount).padStart(8)}
├─────────────────────────────────────────────────────┤
│ PASS: ${n(passCount).padStart(8)}
│ WARNING: ${n(warnCount).padStart(8)}
│ FAIL: ${n(failCount).padStart(8)}
├─────────────────────────────────────────────────────┤
│ Avg security score:${avgScore.toString().padStart(8)}
└─────────────────────────────────────────────────────┘
`);
const dur = ((Date.now() - t0) / 1000).toFixed(1);
console.log(`Completed in ${dur}s`);
await client.end().catch(() => {});
}
main().catch(async e => {
console.error('FAILED:', e.message || e);
await client?.end().catch(() => {});
process.exit(1);
});

View File

@@ -51,6 +51,9 @@ CREATE TABLE IF NOT EXISTS skills (
is_blocked BOOLEAN DEFAULT FALSE, -- Blocked from re-indexing (owner requested removal)
last_scanned TIMESTAMP WITH TIME ZONE,
-- Review pipeline
review_status TEXT DEFAULT 'unreviewed', -- 'unreviewed', 'auto-scored', 'ai-reviewed', 'verified', 'needs-re-review'
-- Content
content_hash TEXT,
raw_content TEXT,

View File

@@ -44,6 +44,11 @@ RUN adduser --system --uid 1001 indexer
# Copy only the bundled dist (no node_modules needed - everything is bundled)
COPY --from=builder --chown=indexer:nodejs /app/services/indexer/dist ./dist
# Curation scripts + skillhub-core (for batch-security, batch-score, curate)
COPY --from=builder --chown=indexer:nodejs /app/scripts/curation ./scripts/curation
COPY --from=builder --chown=indexer:nodejs /app/packages/core/dist ./packages/core/dist
RUN npm install --no-save pg gray-matter yaml
USER indexer
CMD ["node", "dist/worker.js"]