sync: malware page, security alerts, review diagnostics, skill detail improvements
- Add malware security advisory page and SecurityAlertBanner component - Add review diagnostics API and reviewed stats page - Improve skill detail page with better scoring display and metadata - Update review API endpoints with enhanced filtering and stats - Add skill file serving improvements and cache enhancements - Remove legacy curation scripts (moved to internal tooling) - Update CLI search with score filtering support Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
1
.gitignore
vendored
1
.gitignore
vendored
@@ -54,3 +54,4 @@ scripts/curation/data/*
|
|||||||
# !.claude/agents/
|
# !.claude/agents/
|
||||||
# !.claude/council_reasoning.md
|
# !.claude/council_reasoning.md
|
||||||
.claude/settings.json
|
.claude/settings.json
|
||||||
|
.mcp.json
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "skillhub",
|
"name": "skillhub",
|
||||||
"version": "0.2.9",
|
"version": "0.2.11",
|
||||||
"description": "CLI tool for managing AI Agent skills - search, install, and update skills for Claude, Codex, Copilot, and more",
|
"description": "CLI tool for managing AI Agent skills - search, install, and update skills for Claude, Codex, Copilot, and more",
|
||||||
"author": "SkillHub Contributors",
|
"author": "SkillHub Contributors",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
|
|||||||
@@ -39,6 +39,14 @@ export async function install(skillId: string, options: InstallOptions): Promise
|
|||||||
try {
|
try {
|
||||||
skillInfo = await getSkill(skillId);
|
skillInfo = await getSkill(skillId);
|
||||||
if (skillInfo) {
|
if (skillInfo) {
|
||||||
|
// Block installation of malicious skills
|
||||||
|
if (skillInfo.isMalicious) {
|
||||||
|
spinner.fail(chalk.red('This skill has been flagged as malicious (contains malware).'));
|
||||||
|
console.log(chalk.red('Installation blocked for your safety.'));
|
||||||
|
console.log(chalk.dim(`See: https://skills.palebluedot.live/en/skill/${skillId}`));
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
const reviewBadge = skillInfo.aiScore && skillInfo.reviewStatus === 'ai-reviewed'
|
const reviewBadge = skillInfo.aiScore && skillInfo.reviewStatus === 'ai-reviewed'
|
||||||
? chalk.dim(` | AI: ${skillInfo.aiScore}/100`)
|
? chalk.dim(` | AI: ${skillInfo.aiScore}/100`)
|
||||||
: skillInfo.reviewStatus === 'verified'
|
: skillInfo.reviewStatus === 'verified'
|
||||||
|
|||||||
@@ -57,14 +57,15 @@ export async function search(query: string, options: SearchOptions): Promise<voi
|
|||||||
`${num} ${verified} ${chalk.cyan(skill.id.padEnd(38))} ${security}`
|
`${num} ${verified} ${chalk.cyan(skill.id.padEnd(38))} ${security}`
|
||||||
);
|
);
|
||||||
|
|
||||||
// Second line: AI score (if reviewed) + downloads + stars + description
|
// Second line: AI score + downloads + stars + description
|
||||||
const aiScore = skill.aiScore;
|
const aiScore = skill.aiScore;
|
||||||
const hasAiScore = aiScore != null && aiScore > 0 && skill.reviewStatus && skill.reviewStatus !== 'unreviewed' && skill.reviewStatus !== 'auto-scored';
|
const isAiReviewed = aiScore != null && aiScore > 0 && skill.reviewStatus && skill.reviewStatus !== 'unreviewed' && skill.reviewStatus !== 'auto-scored';
|
||||||
const aiPrefix = hasAiScore
|
const showAiScore = isAiReviewed;
|
||||||
? `${chalk.magenta('AI')} ${(aiScore >= 75 ? chalk.green : aiScore >= 50 ? chalk.yellow : chalk.dim)(String(aiScore))} `
|
const aiPrefix = showAiScore
|
||||||
|
? `${chalk.magenta('AI')} ${(aiScore >= 75 ? chalk.green : aiScore >= 50 ? chalk.yellow : chalk.dim)(String(aiScore).padStart(2))} `
|
||||||
: '';
|
: '';
|
||||||
console.log(
|
console.log(
|
||||||
` ${aiPrefix}⬇ ${formatNumber(skill.downloadCount).padStart(6)} ⭐ ${formatNumber(skill.githubStars).padStart(6)} ${chalk.dim(skill.description.slice(0, hasAiScore ? 45 : 55))}${skill.description.length > (hasAiScore ? 45 : 55) ? '...' : ''}`
|
` ${aiPrefix}⬇ ${formatNumber(skill.downloadCount).padStart(6)} ⭐ ${formatNumber(skill.githubStars).padStart(6)} ${chalk.dim(skill.description.slice(0, showAiScore ? 45 : 55))}${skill.description.length > (showAiScore ? 45 : 55) ? '...' : ''}`
|
||||||
);
|
);
|
||||||
|
|
||||||
// Third line: Rating (only if ratingCount >= 3)
|
// Third line: Rating (only if ratingCount >= 3)
|
||||||
|
|||||||
@@ -128,6 +128,7 @@ export interface SkillInfo {
|
|||||||
ratingCount?: number | null;
|
ratingCount?: number | null;
|
||||||
reviewStatus?: string | null;
|
reviewStatus?: string | null;
|
||||||
aiScore?: number | null;
|
aiScore?: number | null;
|
||||||
|
isMalicious?: boolean;
|
||||||
isVerified: boolean;
|
isVerified: boolean;
|
||||||
compatibility: {
|
compatibility: {
|
||||||
platforms: string[];
|
platforms: string[];
|
||||||
|
|||||||
@@ -120,7 +120,7 @@ export default async function CliDocsPage({
|
|||||||
<div>
|
<div>
|
||||||
<p className="text-sm text-text-muted mb-1">{tContent('exSearchSort')}</p>
|
<p className="text-sm text-text-muted mb-1">{tContent('exSearchSort')}</p>
|
||||||
<code className="block text-sm font-mono">
|
<code className="block text-sm font-mono">
|
||||||
<span className="text-text-muted">$</span> npx skillhub search "code review" --sort stars --limit 5
|
<span className="text-text-muted">$</span> npx skillhub search "code review" --sort aiScore --limit 5
|
||||||
</code>
|
</code>
|
||||||
<code className="block text-sm font-mono text-text-muted mt-1">
|
<code className="block text-sm font-mono text-text-muted mt-1">
|
||||||
# Sort options: recommended (default), aiScore, downloads, stars, rating, recent
|
# Sort options: recommended (default), aiScore, downloads, stars, rating, recent
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import { notFound } from 'next/navigation';
|
|||||||
import { locales, localeDirection, type Locale } from '@/i18n';
|
import { locales, localeDirection, type Locale } from '@/i18n';
|
||||||
import { Providers } from '../providers';
|
import { Providers } from '../providers';
|
||||||
import { Suspense } from 'react';
|
import { Suspense } from 'react';
|
||||||
|
import { SecurityAlertBanner } from '@/components/SecurityAlertBanner';
|
||||||
import { QueryNotification } from '@/components/QueryNotification';
|
import { QueryNotification } from '@/components/QueryNotification';
|
||||||
import { ProgressBar } from '@/components/ProgressBar';
|
import { ProgressBar } from '@/components/ProgressBar';
|
||||||
import '../globals.css';
|
import '../globals.css';
|
||||||
@@ -68,6 +69,7 @@ export default async function LocaleLayout({
|
|||||||
<body className="min-h-screen bg-surface">
|
<body className="min-h-screen bg-surface">
|
||||||
<Providers>
|
<Providers>
|
||||||
<NextIntlClientProvider messages={messages}>
|
<NextIntlClientProvider messages={messages}>
|
||||||
|
<SecurityAlertBanner />
|
||||||
<Suspense fallback={null}>
|
<Suspense fallback={null}>
|
||||||
<QueryNotification />
|
<QueryNotification />
|
||||||
</Suspense>
|
</Suspense>
|
||||||
|
|||||||
149
apps/web/app/[locale]/malware/page.tsx
Normal file
149
apps/web/app/[locale]/malware/page.tsx
Normal file
@@ -0,0 +1,149 @@
|
|||||||
|
import { getTranslations, setRequestLocale } from 'next-intl/server';
|
||||||
|
import Link from 'next/link';
|
||||||
|
import { Header } from '@/components/Header';
|
||||||
|
import { Footer } from '@/components/Footer';
|
||||||
|
import { createDb, skillReviewQueries } from '@skillhub/db';
|
||||||
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
|
import { getOrSetCache, cacheKeys, cacheTTL } from '@/lib/cache';
|
||||||
|
import { ShieldAlert, Ban, ExternalLink } from 'lucide-react';
|
||||||
|
|
||||||
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
|
async function getMalwareData(page: number, limit: number) {
|
||||||
|
try {
|
||||||
|
return await getOrSetCache(cacheKeys.malwarePage(page), cacheTTL.malware, async () => {
|
||||||
|
const db = createDb();
|
||||||
|
const offset = (page - 1) * limit;
|
||||||
|
const [skills, total] = await Promise.all([
|
||||||
|
skillReviewQueries.getMaliciousSkills(db, limit, offset),
|
||||||
|
skillReviewQueries.countMaliciousSkills(db),
|
||||||
|
]);
|
||||||
|
return { skills, total };
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error fetching malware skills:', error);
|
||||||
|
return { skills: [], total: 0 };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function generateMetadata({
|
||||||
|
params,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string }>;
|
||||||
|
}) {
|
||||||
|
const { locale } = await params;
|
||||||
|
return {
|
||||||
|
alternates: getPageAlternates(locale, '/malware'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export default async function MalwarePage({
|
||||||
|
params,
|
||||||
|
searchParams,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string }>;
|
||||||
|
searchParams: Promise<{ page?: string }>;
|
||||||
|
}) {
|
||||||
|
const { locale } = await params;
|
||||||
|
const searchParamsResolved = await searchParams;
|
||||||
|
setRequestLocale(locale);
|
||||||
|
const t = await getTranslations('malwarePage');
|
||||||
|
const tMalicious = await getTranslations('malicious');
|
||||||
|
|
||||||
|
const limit = 20;
|
||||||
|
const page = parseInt(searchParamsResolved.page || '1');
|
||||||
|
const { skills: malwareSkills } = await getMalwareData(page, limit);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="min-h-screen flex flex-col">
|
||||||
|
<Header />
|
||||||
|
<main className="flex-1">
|
||||||
|
{/* Header */}
|
||||||
|
<section className="section-header bg-gradient-subtle">
|
||||||
|
<div className="container-main text-center">
|
||||||
|
<div className="inline-flex items-center justify-center w-14 h-14 rounded-2xl bg-error/10 text-error mb-4">
|
||||||
|
<ShieldAlert className="w-7 h-7" />
|
||||||
|
</div>
|
||||||
|
<h1 className="hero-title mb-4">{t('title')}</h1>
|
||||||
|
<p className="hero-subtitle max-w-2xl mx-auto">{t('subtitle')}</p>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section className="section bg-surface">
|
||||||
|
<div className="container-main max-w-4xl">
|
||||||
|
{malwareSkills.length === 0 ? (
|
||||||
|
<p className="text-center text-text-muted py-12">{t('noResults')}</p>
|
||||||
|
) : (
|
||||||
|
<div className="space-y-4">
|
||||||
|
{malwareSkills.map((skill) => (
|
||||||
|
<div
|
||||||
|
key={skill.id}
|
||||||
|
className="card p-5 border-error/20 bg-error/[0.02]"
|
||||||
|
>
|
||||||
|
<div className="flex flex-col sm:flex-row sm:items-start gap-4">
|
||||||
|
<div className="flex-1 min-w-0">
|
||||||
|
<div className="flex items-center gap-2 mb-2 flex-wrap">
|
||||||
|
<Link
|
||||||
|
href={`/${locale}/skill/${skill.id}`}
|
||||||
|
className="text-lg font-semibold text-text-primary hover:text-primary-600 transition-colors truncate"
|
||||||
|
>
|
||||||
|
{skill.name || skill.id}
|
||||||
|
</Link>
|
||||||
|
<span className="flex items-center gap-1 px-1.5 py-0.5 text-xs font-bold rounded border text-error bg-error/10 border-error/20 shrink-0">
|
||||||
|
<ShieldAlert className="w-3 h-3" />
|
||||||
|
{tMalicious('badge')}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<p className="text-sm text-text-muted mb-2 font-mono">
|
||||||
|
{skill.githubOwner}/{skill.githubRepo}
|
||||||
|
</p>
|
||||||
|
|
||||||
|
{skill.description && (
|
||||||
|
<p className="text-sm text-text-secondary line-clamp-2 mb-3">
|
||||||
|
{skill.description}
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<div className="flex items-center gap-4 text-xs text-text-muted flex-wrap">
|
||||||
|
<span className="flex items-center gap-1">
|
||||||
|
<Ban className="w-3.5 h-3.5 text-error" />
|
||||||
|
{t('downloadBlocked')}
|
||||||
|
</span>
|
||||||
|
<span className="flex items-center gap-1">
|
||||||
|
<Ban className="w-3.5 h-3.5 text-error" />
|
||||||
|
{t('installBlocked')}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<a
|
||||||
|
href={`https://github.com/${skill.githubOwner}/${skill.githubRepo}`}
|
||||||
|
target="_blank"
|
||||||
|
rel="noopener noreferrer"
|
||||||
|
className="flex items-center gap-1 text-sm text-text-muted hover:text-text-primary transition-colors shrink-0"
|
||||||
|
>
|
||||||
|
GitHub <ExternalLink className="w-3.5 h-3.5" />
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Back to stats link */}
|
||||||
|
<div className="text-center mt-8">
|
||||||
|
<Link
|
||||||
|
href={`/${locale}/reviewed/stats`}
|
||||||
|
className="btn-secondary"
|
||||||
|
>
|
||||||
|
← {locale === 'fa' ? 'بازگشت به آمار بررسیها' : 'Back to Review Statistics'}
|
||||||
|
</Link>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
</main>
|
||||||
|
<Footer />
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -8,7 +8,8 @@ import { SkillCard } from '@/components/SkillCard';
|
|||||||
import { ReviewedSortSelector } from '@/components/ReviewedSortSelector';
|
import { ReviewedSortSelector } from '@/components/ReviewedSortSelector';
|
||||||
import { getPageAlternates } from '@/lib/seo';
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
import { getOrSetCache, cacheKeys, cacheTTL } from '@/lib/cache';
|
import { getOrSetCache, cacheKeys, cacheTTL } from '@/lib/cache';
|
||||||
import { Sparkles } from 'lucide-react';
|
import { Sparkles, BarChart3 } from 'lucide-react';
|
||||||
|
import Link from 'next/link';
|
||||||
|
|
||||||
|
|
||||||
// Force dynamic rendering to fetch fresh data from database
|
// Force dynamic rendering to fetch fresh data from database
|
||||||
@@ -95,9 +96,16 @@ export default async function ReviewedPage({
|
|||||||
</div>
|
</div>
|
||||||
<h1 className="hero-title mb-4">{t('title')}</h1>
|
<h1 className="hero-title mb-4">{t('title')}</h1>
|
||||||
<p className="hero-subtitle max-w-2xl mx-auto mb-6">{t('subtitle')}</p>
|
<p className="hero-subtitle max-w-2xl mx-auto mb-6">{t('subtitle')}</p>
|
||||||
<p className="text-text-secondary text-sm max-w-3xl mx-auto leading-relaxed">
|
<p className="text-text-secondary text-sm max-w-3xl mx-auto leading-relaxed mb-4">
|
||||||
{t('explanation')}
|
{t('explanation')}
|
||||||
</p>
|
</p>
|
||||||
|
<Link
|
||||||
|
href={`/${locale}/reviewed/stats`}
|
||||||
|
className="inline-flex items-center gap-1.5 text-sm text-primary-600 hover:text-primary-700 font-medium"
|
||||||
|
>
|
||||||
|
<BarChart3 className="w-4 h-4" />
|
||||||
|
{locale === 'fa' ? 'مشاهده آمار بررسیها' : 'View Review Statistics'}
|
||||||
|
</Link>
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
|
|||||||
224
apps/web/app/[locale]/reviewed/stats/page.tsx
Normal file
224
apps/web/app/[locale]/reviewed/stats/page.tsx
Normal file
@@ -0,0 +1,224 @@
|
|||||||
|
import { getTranslations, setRequestLocale } from 'next-intl/server';
|
||||||
|
import Link from 'next/link';
|
||||||
|
import { Header } from '@/components/Header';
|
||||||
|
import { Footer } from '@/components/Footer';
|
||||||
|
import { createDb, skillReviewQueries } from '@skillhub/db';
|
||||||
|
import { getPageAlternates } from '@/lib/seo';
|
||||||
|
import { getOrSetCache, cacheKeys, cacheTTL } from '@/lib/cache';
|
||||||
|
import { BarChart3, ShieldAlert, AlertTriangle, CheckCircle } from 'lucide-react';
|
||||||
|
|
||||||
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
|
interface StatsData {
|
||||||
|
pipeline: Record<string, number>;
|
||||||
|
totalReviews: number;
|
||||||
|
scoreDistribution: Record<string, number>;
|
||||||
|
securityStats: Record<string, number>;
|
||||||
|
malwareCount: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getPublicReviewStats(): Promise<StatsData | null> {
|
||||||
|
try {
|
||||||
|
return await getOrSetCache(cacheKeys.reviewStatsPublic(), cacheTTL.reviewStatsPublic, async () => {
|
||||||
|
const db = createDb();
|
||||||
|
const [pipeline, totalReviews, scoreDistribution, securityStats, malwareCount] = await Promise.all([
|
||||||
|
skillReviewQueries.getPublicPipelineStats(db),
|
||||||
|
skillReviewQueries.countTotalReviews(db),
|
||||||
|
skillReviewQueries.getScoreDistribution(db),
|
||||||
|
skillReviewQueries.getSecurityStats(db),
|
||||||
|
skillReviewQueries.countMaliciousSkills(db),
|
||||||
|
]);
|
||||||
|
return { pipeline, totalReviews, scoreDistribution, securityStats, malwareCount };
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error fetching review stats:', error);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function generateMetadata({
|
||||||
|
params,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string }>;
|
||||||
|
}) {
|
||||||
|
const { locale } = await params;
|
||||||
|
return {
|
||||||
|
alternates: getPageAlternates(locale, '/reviewed/stats'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function StatCard({
|
||||||
|
icon: Icon,
|
||||||
|
label,
|
||||||
|
value,
|
||||||
|
color,
|
||||||
|
href,
|
||||||
|
}: {
|
||||||
|
icon: React.ElementType;
|
||||||
|
label: string;
|
||||||
|
value: number;
|
||||||
|
color: string;
|
||||||
|
href?: string;
|
||||||
|
}) {
|
||||||
|
const content = (
|
||||||
|
<div className={`flex items-center gap-3 p-4 rounded-xl border border-border bg-surface ${href ? 'hover:border-primary-300 transition-colors' : ''}`}>
|
||||||
|
<div className={`flex items-center justify-center w-10 h-10 rounded-lg ${color}`}>
|
||||||
|
<Icon className="w-5 h-5" />
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<p className="text-2xl font-bold text-text-primary ltr-nums">{value.toLocaleString()}</p>
|
||||||
|
<p className="text-sm text-text-secondary">{label}</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
|
||||||
|
if (href) {
|
||||||
|
return <Link href={href}>{content}</Link>;
|
||||||
|
}
|
||||||
|
return content;
|
||||||
|
}
|
||||||
|
|
||||||
|
function BarRow({
|
||||||
|
label,
|
||||||
|
value,
|
||||||
|
total,
|
||||||
|
color,
|
||||||
|
}: {
|
||||||
|
label: string;
|
||||||
|
value: number;
|
||||||
|
total: number;
|
||||||
|
color: string;
|
||||||
|
}) {
|
||||||
|
const pct = total > 0 ? (value / total) * 100 : 0;
|
||||||
|
return (
|
||||||
|
<div className="space-y-1">
|
||||||
|
<div className="flex items-center justify-between text-sm">
|
||||||
|
<span className="text-text-secondary">{label}</span>
|
||||||
|
<span className="font-medium text-text-primary ltr-nums">
|
||||||
|
{value.toLocaleString()} <span className="text-text-muted">({pct.toFixed(1)}%)</span>
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<div className="h-3 rounded-full bg-surface-subtle overflow-hidden">
|
||||||
|
<div
|
||||||
|
className={`h-full rounded-full ${color} transition-all duration-500`}
|
||||||
|
style={{ width: `${Math.max(pct, 0.5)}%` }}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export default async function ReviewStatsPage({
|
||||||
|
params,
|
||||||
|
}: {
|
||||||
|
params: Promise<{ locale: string }>;
|
||||||
|
}) {
|
||||||
|
const { locale } = await params;
|
||||||
|
setRequestLocale(locale);
|
||||||
|
const t = await getTranslations('reviewStats');
|
||||||
|
|
||||||
|
const data = await getPublicReviewStats();
|
||||||
|
|
||||||
|
if (!data) {
|
||||||
|
return (
|
||||||
|
<div className="min-h-screen flex flex-col">
|
||||||
|
<Header />
|
||||||
|
<main className="flex-1 flex items-center justify-center">
|
||||||
|
<p className="text-text-muted">Failed to load stats.</p>
|
||||||
|
</main>
|
||||||
|
<Footer />
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Total = all browse-ready SKILL.md skills (for meaningful percentages)
|
||||||
|
const pipelineTotal = Object.values(data.pipeline).reduce((sum, n) => sum + n, 0);
|
||||||
|
|
||||||
|
const scoreTotal =
|
||||||
|
data.scoreDistribution.high +
|
||||||
|
data.scoreDistribution.mid +
|
||||||
|
data.scoreDistribution.low;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="min-h-screen flex flex-col">
|
||||||
|
<Header />
|
||||||
|
<main className="flex-1">
|
||||||
|
{/* Header Section */}
|
||||||
|
<section className="section-header bg-gradient-subtle">
|
||||||
|
<div className="container-main text-center">
|
||||||
|
<div className="inline-flex items-center justify-center w-14 h-14 rounded-2xl bg-primary-50 text-primary-600 mb-4">
|
||||||
|
<BarChart3 className="w-7 h-7" />
|
||||||
|
</div>
|
||||||
|
<h1 className="hero-title mb-4">{t('title')}</h1>
|
||||||
|
<p className="hero-subtitle max-w-2xl mx-auto">{t('subtitle')}</p>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section className="section bg-surface">
|
||||||
|
<div className="container-main max-w-4xl">
|
||||||
|
{/* Review Pipeline */}
|
||||||
|
<div className="card p-6 mb-8">
|
||||||
|
<h2 className="text-xl font-semibold text-text-primary mb-1">{t('pipelineTitle')}</h2>
|
||||||
|
<p className="text-sm text-text-muted mb-6">{t('pipelineDescription')}</p>
|
||||||
|
<div className="space-y-4">
|
||||||
|
<BarRow label={t('aiReviewed')} value={data.pipeline['ai-reviewed'] ?? 0} total={pipelineTotal} color="bg-primary-500" />
|
||||||
|
<BarRow label={t('needsReReview')} value={data.pipeline['needs-re-review'] ?? 0} total={pipelineTotal} color="bg-warning" />
|
||||||
|
</div>
|
||||||
|
<div className="mt-6 pt-4 border-t border-border flex items-center justify-between text-sm">
|
||||||
|
<span className="text-text-secondary">{t('totalReviews')}</span>
|
||||||
|
<span className="font-semibold text-text-primary ltr-nums">{data.totalReviews.toLocaleString()}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Score Distribution */}
|
||||||
|
<div className="card p-6 mb-8">
|
||||||
|
<h2 className="text-xl font-semibold text-text-primary mb-1">{t('scoreTitle')}</h2>
|
||||||
|
<p className="text-sm text-text-muted mb-6">{t('scoreDescription')}</p>
|
||||||
|
<div className="space-y-4">
|
||||||
|
<BarRow label={t('scoreHigh')} value={data.scoreDistribution.high} total={scoreTotal} color="bg-success" />
|
||||||
|
<BarRow label={t('scoreMid')} value={data.scoreDistribution.mid} total={scoreTotal} color="bg-gold" />
|
||||||
|
<BarRow label={t('scoreLow')} value={data.scoreDistribution.low} total={scoreTotal} color="bg-error" />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Security & Malware */}
|
||||||
|
<div className="grid grid-cols-1 md:grid-cols-2 gap-8">
|
||||||
|
{/* Security Scan */}
|
||||||
|
<div className="card p-6">
|
||||||
|
<h2 className="text-xl font-semibold text-text-primary mb-1">{t('securityTitle')}</h2>
|
||||||
|
<p className="text-sm text-text-muted mb-6">{t('securityDescription')}</p>
|
||||||
|
<div className="space-y-3">
|
||||||
|
<StatCard icon={CheckCircle} label={t('securityPass')} value={data.securityStats.pass} color="bg-success/10 text-success" />
|
||||||
|
<StatCard icon={AlertTriangle} label={t('securityWarning')} value={data.securityStats.warning} color="bg-warning/10 text-warning" />
|
||||||
|
<StatCard icon={ShieldAlert} label={t('securityFail')} value={data.securityStats.fail} color="bg-error/10 text-error" />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Malware Detection */}
|
||||||
|
<div className="card p-6">
|
||||||
|
<h2 className="text-xl font-semibold text-text-primary mb-1">{t('malwareTitle')}</h2>
|
||||||
|
<p className="text-sm text-text-muted mb-6">{t('malwareDescription')}</p>
|
||||||
|
<StatCard
|
||||||
|
icon={ShieldAlert}
|
||||||
|
label={t('malwareFlagged')}
|
||||||
|
value={data.malwareCount}
|
||||||
|
color="bg-error/10 text-error"
|
||||||
|
href={`/${locale}/malware`}
|
||||||
|
/>
|
||||||
|
{data.malwareCount > 0 && (
|
||||||
|
<Link
|
||||||
|
href={`/${locale}/malware`}
|
||||||
|
className="inline-flex items-center gap-1.5 mt-4 text-sm text-primary-600 hover:text-primary-700 font-medium"
|
||||||
|
>
|
||||||
|
{t('malwareViewAll')} →
|
||||||
|
</Link>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
</main>
|
||||||
|
<Footer />
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -4,7 +4,7 @@ import { notFound } from 'next/navigation';
|
|||||||
import { headers } from 'next/headers';
|
import { headers } from 'next/headers';
|
||||||
import {
|
import {
|
||||||
Star, Download, Shield, CheckCircle, Copy,
|
Star, Download, Shield, CheckCircle, Copy,
|
||||||
ExternalLink, Github, Calendar, User, Tag, ChevronRight, Eye, Sparkles
|
ExternalLink, Github, Calendar, User, Tag, ChevronRight, Eye, Sparkles, XCircle, ShieldAlert
|
||||||
} from 'lucide-react';
|
} from 'lucide-react';
|
||||||
import { Header } from '@/components/Header';
|
import { Header } from '@/components/Header';
|
||||||
import { Footer } from '@/components/Footer';
|
import { Footer } from '@/components/Footer';
|
||||||
@@ -113,6 +113,8 @@ export default async function SkillPage({ params }: SkillPageProps) {
|
|||||||
notFound();
|
notFound();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const isMalicious = dbSkill.isMalicious ?? false;
|
||||||
|
|
||||||
// Track view count with IP-based rate limiting (1 hour cooldown per IP)
|
// Track view count with IP-based rate limiting (1 hour cooldown per IP)
|
||||||
// Get client IP from headers (works with Cloudflare, nginx, etc.)
|
// Get client IP from headers (works with Cloudflare, nginx, etc.)
|
||||||
const headersList = await headers();
|
const headersList = await headers();
|
||||||
@@ -159,7 +161,13 @@ export default async function SkillPage({ params }: SkillPageProps) {
|
|||||||
|
|
||||||
// Parse review notes for structured display
|
// Parse review notes for structured display
|
||||||
const parsedNotes = review?.reviewNotes ? parseReviewNotes(review.reviewNotes) : null;
|
const parsedNotes = review?.reviewNotes ? parseReviewNotes(review.reviewNotes) : null;
|
||||||
const hasReview = review && review.aiScore && (dbSkill.reviewStatus === 'ai-reviewed' || dbSkill.reviewStatus === 'verified');
|
const hasReview = review && review.aiScore != null && (dbSkill.reviewStatus === 'ai-reviewed' || dbSkill.reviewStatus === 'verified');
|
||||||
|
const isRejected = hasReview && review.aiScore === 0;
|
||||||
|
// Review is outdated only if both hashes are non-empty and differ
|
||||||
|
const reviewOutdated = hasReview
|
||||||
|
&& review.contentHashAtReview && review.contentHashAtReview.length > 1
|
||||||
|
&& dbSkill.contentHash && dbSkill.contentHash.length > 1
|
||||||
|
&& dbSkill.contentHash !== review.contentHashAtReview;
|
||||||
|
|
||||||
// Content section title based on source format (uses FORMAT_LABELS from skillhub-core)
|
// Content section title based on source format (uses FORMAT_LABELS from skillhub-core)
|
||||||
const getContentTitle = (format: string) => {
|
const getContentTitle = (format: string) => {
|
||||||
@@ -358,12 +366,23 @@ export default async function SkillPage({ params }: SkillPageProps) {
|
|||||||
|
|
||||||
{/* Compact AI Review Score */}
|
{/* Compact AI Review Score */}
|
||||||
{hasReview && (
|
{hasReview && (
|
||||||
<div className="flex items-center gap-3 px-4 py-3 bg-surface-elevated rounded-xl border border-border">
|
<div className={`flex items-center gap-3 px-4 py-3 rounded-xl border ${isRejected ? 'bg-error/5 border-error/20' : 'bg-surface-elevated border-border'}`}>
|
||||||
<Sparkles className={`w-5 h-5 ${review.aiScore! >= 75 ? 'text-success' : 'text-gold'}`} />
|
{isRejected ? (
|
||||||
<span className={`text-2xl font-bold ltr-nums ${review.aiScore! >= 75 ? 'text-success' : review.aiScore! >= 50 ? 'text-gold' : 'text-text-primary'}`}>
|
<>
|
||||||
{review.aiScore}
|
<XCircle className="w-5 h-5 text-error" />
|
||||||
</span>
|
<span className="text-lg font-bold text-error">
|
||||||
<span className="text-sm text-text-muted">{t('review.outOf100')}</span>
|
{isRTL ? 'رد شده' : 'Rejected'}
|
||||||
|
</span>
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
<Sparkles className={`w-5 h-5 ${review.aiScore! >= 75 ? 'text-success' : review.aiScore! >= 50 ? 'text-gold' : 'text-text-muted'}`} />
|
||||||
|
<span className={`text-2xl font-bold ltr-nums ${review.aiScore! >= 75 ? 'text-success' : review.aiScore! >= 50 ? 'text-gold' : 'text-text-primary'}`}>
|
||||||
|
{review.aiScore}
|
||||||
|
</span>
|
||||||
|
<span className="text-sm text-text-muted">{t('review.outOf100')}</span>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
@@ -396,6 +415,25 @@ export default async function SkillPage({ params }: SkillPageProps) {
|
|||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
{/* Malicious Skill Warning Banner */}
|
||||||
|
{isMalicious && (
|
||||||
|
<div className="bg-error/10 border-y-2 border-error px-4 py-6">
|
||||||
|
<div className="container-main flex items-start gap-4">
|
||||||
|
<ShieldAlert className="w-8 h-8 text-error flex-shrink-0 mt-0.5" />
|
||||||
|
<div>
|
||||||
|
<h2 className="text-lg font-bold text-error mb-1" dir="auto">
|
||||||
|
{isRTL ? 'بدافزار شناسایی شد' : 'Malware Detected'}
|
||||||
|
</h2>
|
||||||
|
<p className="text-text-secondary text-sm" dir="auto">
|
||||||
|
{isRTL
|
||||||
|
? 'این مهارت به عنوان مخرب شناسایی شده است. شامل کد مبهمسازی شده برای دانلود و اجرای بارهای مضر است. دانلود فایل و نصب مسدود شده است.'
|
||||||
|
: 'This skill has been flagged as malicious. It contains obfuscated code designed to download and execute harmful payloads. File downloads and installation are blocked.'}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
{/* Stale Skill Warning Banner */}
|
{/* Stale Skill Warning Banner */}
|
||||||
{dbSkill.isStale && (
|
{dbSkill.isStale && (
|
||||||
<div className="bg-warning/10 border-b border-warning/20">
|
<div className="bg-warning/10 border-b border-warning/20">
|
||||||
@@ -470,56 +508,87 @@ export default async function SkillPage({ params }: SkillPageProps) {
|
|||||||
<div className="lg:col-span-2 space-y-6">
|
<div className="lg:col-span-2 space-y-6">
|
||||||
{/* Quick Install (Mobile) */}
|
{/* Quick Install (Mobile) */}
|
||||||
<div className="lg:hidden">
|
<div className="lg:hidden">
|
||||||
<InstallSection
|
{isMalicious ? (
|
||||||
skillId={skill.id}
|
<div className="bg-error/5 border border-error/20 rounded-lg p-4 text-center">
|
||||||
skillName={skill.name}
|
<ShieldAlert className="w-6 h-6 text-error mx-auto mb-2" />
|
||||||
repositoryUrl={skill.repository}
|
<p className="text-sm text-error font-medium" dir="auto">
|
||||||
sourceFormat={skill.sourceFormat}
|
{isRTL ? 'نصب مسدود شده — این مهارت حاوی بدافزار است' : 'Installation blocked — this skill contains malware'}
|
||||||
installCommands={installCommands}
|
</p>
|
||||||
translations={{
|
</div>
|
||||||
title: t('install.title'),
|
) : (
|
||||||
cli: t('install.cli'),
|
<InstallSection
|
||||||
cliGlobal: t('install.cliGlobal') || 'Install globally (user-level):',
|
skillId={skill.id}
|
||||||
cliProject: t('install.cliProject') || 'Install in current project:',
|
skillName={skill.name}
|
||||||
selectFolder: t('install.selectFolder'),
|
repositoryUrl={skill.repository}
|
||||||
suggestedPath: t('install.suggestedPath'),
|
sourceFormat={skill.sourceFormat}
|
||||||
copied: t('install.copied'),
|
installCommands={installCommands}
|
||||||
downloadZip: t('install.downloadZip') || 'Download ZIP',
|
translations={{
|
||||||
copyCommand: t('install.copyCommand') || 'Copy command',
|
title: t('install.title'),
|
||||||
downloading: t('install.downloading') || 'Downloading...',
|
cli: t('install.cli'),
|
||||||
installing: t('install.installing') || 'Installing...',
|
cliGlobal: t('install.cliGlobal') || 'Install globally (user-level):',
|
||||||
installed: t('install.installed') || 'Installed!',
|
cliProject: t('install.cliProject') || 'Install in current project:',
|
||||||
downloadFailed: t('install.downloadFailed') || 'Download failed',
|
selectFolder: t('install.selectFolder'),
|
||||||
browserNotSupported: t('install.browserNotSupported') || 'Browser not supported',
|
suggestedPath: t('install.suggestedPath'),
|
||||||
rateLimitError: t('install.rateLimitError') || 'Rate limit exceeded',
|
copied: t('install.copied'),
|
||||||
timeoutError: t('install.timeoutError') || 'Request timed out',
|
downloadZip: t('install.downloadZip') || 'Download ZIP',
|
||||||
notFoundError: t('install.notFoundError') || 'Skill not found',
|
copyCommand: t('install.copyCommand') || 'Copy command',
|
||||||
noFilesError: t('install.noFilesError') || 'No files found',
|
downloading: t('install.downloading') || 'Downloading...',
|
||||||
disclaimer: t('install.disclaimer'),
|
installing: t('install.installing') || 'Installing...',
|
||||||
folderNotePrefix: t('install.folderNotePrefix') || 'A folder named "',
|
installed: t('install.installed') || 'Installed!',
|
||||||
folderNoteSuffix: t('install.folderNoteSuffix') || '" will be created',
|
downloadFailed: t('install.downloadFailed') || 'Download failed',
|
||||||
}}
|
browserNotSupported: t('install.browserNotSupported') || 'Browser not supported',
|
||||||
/>
|
rateLimitError: t('install.rateLimitError') || 'Rate limit exceeded',
|
||||||
|
timeoutError: t('install.timeoutError') || 'Request timed out',
|
||||||
|
notFoundError: t('install.notFoundError') || 'Skill not found',
|
||||||
|
noFilesError: t('install.noFilesError') || 'No files found',
|
||||||
|
disclaimer: t('install.disclaimer'),
|
||||||
|
folderNotePrefix: t('install.folderNotePrefix') || 'A folder named "',
|
||||||
|
folderNoteSuffix: t('install.folderNoteSuffix') || '" will be created',
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* AI Review Card (Mobile) */}
|
{/* AI Review Card (Mobile) */}
|
||||||
{hasReview && (
|
{hasReview && (
|
||||||
<div className="lg:hidden bg-surface-elevated rounded-2xl border border-border p-6">
|
<div className={`lg:hidden rounded-2xl border p-6 ${isRejected ? 'bg-error/5 border-error/20' : 'bg-surface-elevated border-border'}`}>
|
||||||
<h3 className="font-semibold text-text-primary mb-4 flex items-center gap-2">
|
<h3 className="font-semibold text-text-primary mb-4 flex items-center gap-2">
|
||||||
<Sparkles className="w-4 h-4 text-primary-500" />
|
{isRejected ? <XCircle className="w-4 h-4 text-error" /> : <Sparkles className="w-4 h-4 text-primary-500" />}
|
||||||
{t('review.title')}
|
{t('review.title')}
|
||||||
</h3>
|
</h3>
|
||||||
<div className="space-y-4">
|
{isRejected ? (
|
||||||
<div className="space-y-3">
|
<div className="space-y-4">
|
||||||
<ScoreBar label={t('review.instructionQuality')} score={review.instructionQuality} />
|
<div className="text-center">
|
||||||
<ScoreBar label={t('review.descriptionPrecision')} score={review.descriptionPrecision} />
|
<div className="text-xl font-bold text-error">
|
||||||
<ScoreBar label={t('review.usefulness')} score={review.usefulness} />
|
{isRTL ? 'رد شده' : 'Rejected'}
|
||||||
<ScoreBar label={t('review.technicalSoundness')} score={review.technicalSoundness} />
|
</div>
|
||||||
|
<div className="text-sm text-text-muted">
|
||||||
|
{isRTL ? 'این مهارت معیارهای کیفیت را ندارد' : 'Does not meet quality standards'}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{parsedNotes?.rationale && (
|
||||||
|
<p className="text-sm text-text-secondary pt-3 border-t border-error/20" dir="auto">{parsedNotes.rationale}</p>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
{parsedNotes?.rationale && (
|
) : (
|
||||||
<p className="text-sm text-text-secondary pt-3 border-t border-border" dir="auto">{parsedNotes.rationale}</p>
|
<div className="space-y-4">
|
||||||
)}
|
<div className="space-y-3">
|
||||||
</div>
|
<ScoreBar label={t('review.instructionQuality')} score={review.instructionQuality} />
|
||||||
|
<ScoreBar label={t('review.descriptionPrecision')} score={review.descriptionPrecision} />
|
||||||
|
<ScoreBar label={t('review.usefulness')} score={review.usefulness} />
|
||||||
|
<ScoreBar label={t('review.technicalSoundness')} score={review.technicalSoundness} />
|
||||||
|
</div>
|
||||||
|
{parsedNotes?.rationale && (
|
||||||
|
<p className="text-sm text-text-secondary pt-3 border-t border-border" dir="auto">{parsedNotes.rationale}</p>
|
||||||
|
)}
|
||||||
|
{reviewOutdated && (
|
||||||
|
<p className="text-xs text-warning flex items-center gap-1.5 pt-3 border-t border-border">
|
||||||
|
<span>⚠</span>
|
||||||
|
{isRTL ? 'بررسی بر اساس نسخه قبلی' : 'Review based on previous version'}
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
@@ -547,61 +616,84 @@ export default async function SkillPage({ params }: SkillPageProps) {
|
|||||||
<div className="sticky top-24 max-h-[calc(100vh-7rem)] overflow-y-auto space-y-6 scrollbar-thin">
|
<div className="sticky top-24 max-h-[calc(100vh-7rem)] overflow-y-auto space-y-6 scrollbar-thin">
|
||||||
{/* AI Review Card (Desktop) — above Install for visibility */}
|
{/* AI Review Card (Desktop) — above Install for visibility */}
|
||||||
{hasReview && (
|
{hasReview && (
|
||||||
<div className="bg-surface-elevated rounded-2xl border border-border p-6">
|
<div className={`rounded-2xl border p-6 ${isRejected ? 'bg-error/5 border-error/20' : 'bg-surface-elevated border-border'}`}>
|
||||||
<h3 className="font-semibold text-text-primary mb-4 flex items-center gap-2">
|
<h3 className="font-semibold text-text-primary mb-4 flex items-center gap-2">
|
||||||
<Sparkles className="w-4 h-4 text-primary-500" />
|
{isRejected ? <XCircle className="w-4 h-4 text-error" /> : <Sparkles className="w-4 h-4 text-primary-500" />}
|
||||||
{t('review.title')}
|
{t('review.title')}
|
||||||
</h3>
|
</h3>
|
||||||
|
|
||||||
{/* Overall Score */}
|
{isRejected ? (
|
||||||
<div className="text-center mb-4">
|
<>
|
||||||
<div className={`text-4xl font-bold ${review.aiScore! >= 75 ? 'text-success' : review.aiScore! >= 50 ? 'text-gold' : 'text-text-primary'}`}>
|
{/* Rejected status */}
|
||||||
{review.aiScore}
|
<div className="text-center mb-4">
|
||||||
</div>
|
<div className="text-2xl font-bold text-error">
|
||||||
<div className="text-sm text-text-muted">{t('review.outOf100')}</div>
|
{isRTL ? 'رد شده' : 'Rejected'}
|
||||||
</div>
|
</div>
|
||||||
|
<div className="text-sm text-text-muted">
|
||||||
{/* 4-axis score bars */}
|
{isRTL ? 'این مهارت معیارهای کیفیت را ندارد' : 'Does not meet quality standards'}
|
||||||
<div className="space-y-3">
|
</div>
|
||||||
<ScoreBar label={t('review.instructionQuality')} score={review.instructionQuality} />
|
|
||||||
<ScoreBar label={t('review.descriptionPrecision')} score={review.descriptionPrecision} />
|
|
||||||
<ScoreBar label={t('review.usefulness')} score={review.usefulness} />
|
|
||||||
<ScoreBar label={t('review.technicalSoundness')} score={review.technicalSoundness} />
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{/* Rationale */}
|
|
||||||
{parsedNotes?.rationale && (
|
|
||||||
<div className="mt-4 pt-4 border-t border-border">
|
|
||||||
<p className="text-sm text-text-secondary" dir="auto">{parsedNotes.rationale}</p>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{/* Tags: Audience, Maturity, Complexity, Use Cases */}
|
|
||||||
{(parsedNotes?.maturity || parsedNotes?.complexity || (parsedNotes?.audience && parsedNotes.audience.length > 0) || (parsedNotes?.useCases && parsedNotes.useCases.length > 0)) && (
|
|
||||||
<div className="mt-4 pt-4 border-t border-border">
|
|
||||||
<div className="flex flex-wrap gap-1.5">
|
|
||||||
{parsedNotes?.maturity && (
|
|
||||||
<span className="px-2 py-0.5 text-xs rounded bg-surface-subtle text-text-muted border border-border">
|
|
||||||
{parsedNotes.maturity}
|
|
||||||
</span>
|
|
||||||
)}
|
|
||||||
{parsedNotes?.complexity && (
|
|
||||||
<span className="px-2 py-0.5 text-xs rounded bg-surface-subtle text-text-muted border border-border">
|
|
||||||
{parsedNotes.complexity}
|
|
||||||
</span>
|
|
||||||
)}
|
|
||||||
{parsedNotes?.audience?.map((a: string) => (
|
|
||||||
<span key={a} className="px-2 py-0.5 text-xs rounded bg-primary-50 text-primary-700 dark:bg-primary-900/20 dark:text-primary-400">
|
|
||||||
{a.trim()}
|
|
||||||
</span>
|
|
||||||
))}
|
|
||||||
{parsedNotes?.useCases?.map((uc: string) => (
|
|
||||||
<span key={uc} className="px-2 py-0.5 text-xs rounded bg-success/10 text-success">
|
|
||||||
{uc.trim()}
|
|
||||||
</span>
|
|
||||||
))}
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
|
||||||
|
{/* Rationale for rejection */}
|
||||||
|
{parsedNotes?.rationale && (
|
||||||
|
<div className="pt-4 border-t border-error/20">
|
||||||
|
<p className="text-sm text-text-secondary" dir="auto">{parsedNotes.rationale}</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
{/* Overall Score */}
|
||||||
|
<div className="text-center mb-4">
|
||||||
|
<div className={`text-4xl font-bold ${review.aiScore! >= 75 ? 'text-success' : review.aiScore! >= 50 ? 'text-gold' : 'text-text-primary'}`}>
|
||||||
|
{review.aiScore}
|
||||||
|
</div>
|
||||||
|
<div className="text-sm text-text-muted">{t('review.outOf100')}</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* 4-axis score bars */}
|
||||||
|
<div className="space-y-3">
|
||||||
|
<ScoreBar label={t('review.instructionQuality')} score={review.instructionQuality} />
|
||||||
|
<ScoreBar label={t('review.descriptionPrecision')} score={review.descriptionPrecision} />
|
||||||
|
<ScoreBar label={t('review.usefulness')} score={review.usefulness} />
|
||||||
|
<ScoreBar label={t('review.technicalSoundness')} score={review.technicalSoundness} />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Rationale */}
|
||||||
|
{parsedNotes?.rationale && (
|
||||||
|
<div className="mt-4 pt-4 border-t border-border">
|
||||||
|
<p className="text-sm text-text-secondary" dir="auto">{parsedNotes.rationale}</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Tags: Audience, Maturity, Complexity, Use Cases */}
|
||||||
|
{(parsedNotes?.maturity || parsedNotes?.complexity || (parsedNotes?.audience && parsedNotes.audience.length > 0) || (parsedNotes?.useCases && parsedNotes.useCases.length > 0)) && (
|
||||||
|
<div className="mt-4 pt-4 border-t border-border">
|
||||||
|
<div className="flex flex-wrap gap-1.5">
|
||||||
|
{parsedNotes?.maturity && (
|
||||||
|
<span className="px-2 py-0.5 text-xs rounded bg-surface-subtle text-text-muted border border-border">
|
||||||
|
{parsedNotes.maturity}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
{parsedNotes?.complexity && (
|
||||||
|
<span className="px-2 py-0.5 text-xs rounded bg-surface-subtle text-text-muted border border-border">
|
||||||
|
{parsedNotes.complexity}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
{parsedNotes?.audience?.map((a: string) => (
|
||||||
|
<span key={a} className="px-2 py-0.5 text-xs rounded bg-primary-50 text-primary-700 dark:bg-primary-900/20 dark:text-primary-400">
|
||||||
|
{a.trim()}
|
||||||
|
</span>
|
||||||
|
))}
|
||||||
|
{parsedNotes?.useCases?.map((uc: string) => (
|
||||||
|
<span key={uc} className="px-2 py-0.5 text-xs rounded bg-success/10 text-success">
|
||||||
|
{uc.trim()}
|
||||||
|
</span>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{/* Reviewer info */}
|
{/* Reviewer info */}
|
||||||
@@ -611,40 +703,59 @@ export default async function SkillPage({ params }: SkillPageProps) {
|
|||||||
<> {t('review.reviewedOn', { date: new Date(review.reviewedAt).toLocaleDateString(locale === 'fa' ? 'fa-IR' : 'en-US') })}</>
|
<> {t('review.reviewedOn', { date: new Date(review.reviewedAt).toLocaleDateString(locale === 'fa' ? 'fa-IR' : 'en-US') })}</>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/* Outdated review warning */}
|
||||||
|
{!isRejected && reviewOutdated && (
|
||||||
|
<div className="mt-3 pt-3 border-t border-border">
|
||||||
|
<p className="text-xs text-warning flex items-center gap-1.5">
|
||||||
|
<span>⚠</span>
|
||||||
|
{isRTL ? 'بررسی بر اساس نسخه قبلی' : 'Review based on previous version'}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{/* Install Section */}
|
{/* Install Section */}
|
||||||
<InstallSection
|
{isMalicious ? (
|
||||||
skillId={skill.id}
|
<div className="bg-error/5 border border-error/20 rounded-lg p-4 text-center">
|
||||||
skillName={skill.name}
|
<ShieldAlert className="w-6 h-6 text-error mx-auto mb-2" />
|
||||||
repositoryUrl={skill.repository}
|
<p className="text-sm text-error font-medium" dir="auto">
|
||||||
sourceFormat={skill.sourceFormat}
|
{isRTL ? 'نصب مسدود شده — این مهارت حاوی بدافزار است' : 'Installation blocked — this skill contains malware'}
|
||||||
installCommands={installCommands}
|
</p>
|
||||||
translations={{
|
</div>
|
||||||
title: t('install.title'),
|
) : (
|
||||||
cli: t('install.cli'),
|
<InstallSection
|
||||||
cliGlobal: t('install.cliGlobal') || 'Install globally (user-level):',
|
skillId={skill.id}
|
||||||
cliProject: t('install.cliProject') || 'Install in current project:',
|
skillName={skill.name}
|
||||||
selectFolder: t('install.selectFolder'),
|
repositoryUrl={skill.repository}
|
||||||
suggestedPath: t('install.suggestedPath'),
|
sourceFormat={skill.sourceFormat}
|
||||||
copied: t('install.copied'),
|
installCommands={installCommands}
|
||||||
downloadZip: t('install.downloadZip') || 'Download ZIP',
|
translations={{
|
||||||
copyCommand: t('install.copyCommand') || 'Copy command',
|
title: t('install.title'),
|
||||||
downloading: t('install.downloading') || 'Downloading...',
|
cli: t('install.cli'),
|
||||||
installing: t('install.installing') || 'Installing...',
|
cliGlobal: t('install.cliGlobal') || 'Install globally (user-level):',
|
||||||
installed: t('install.installed') || 'Installed!',
|
cliProject: t('install.cliProject') || 'Install in current project:',
|
||||||
downloadFailed: t('install.downloadFailed') || 'Download failed',
|
selectFolder: t('install.selectFolder'),
|
||||||
browserNotSupported: t('install.browserNotSupported') || 'Browser not supported',
|
suggestedPath: t('install.suggestedPath'),
|
||||||
rateLimitError: t('install.rateLimitError') || 'Rate limit exceeded',
|
copied: t('install.copied'),
|
||||||
timeoutError: t('install.timeoutError') || 'Request timed out',
|
downloadZip: t('install.downloadZip') || 'Download ZIP',
|
||||||
notFoundError: t('install.notFoundError') || 'Skill not found',
|
copyCommand: t('install.copyCommand') || 'Copy command',
|
||||||
noFilesError: t('install.noFilesError') || 'No files found',
|
downloading: t('install.downloading') || 'Downloading...',
|
||||||
disclaimer: t('install.disclaimer'),
|
installing: t('install.installing') || 'Installing...',
|
||||||
folderNotePrefix: t('install.folderNotePrefix') || 'A folder named "',
|
installed: t('install.installed') || 'Installed!',
|
||||||
folderNoteSuffix: t('install.folderNoteSuffix') || '" will be created',
|
downloadFailed: t('install.downloadFailed') || 'Download failed',
|
||||||
}}
|
browserNotSupported: t('install.browserNotSupported') || 'Browser not supported',
|
||||||
/>
|
rateLimitError: t('install.rateLimitError') || 'Rate limit exceeded',
|
||||||
|
timeoutError: t('install.timeoutError') || 'Request timed out',
|
||||||
|
notFoundError: t('install.notFoundError') || 'Skill not found',
|
||||||
|
noFilesError: t('install.noFilesError') || 'No files found',
|
||||||
|
disclaimer: t('install.disclaimer'),
|
||||||
|
folderNotePrefix: t('install.folderNotePrefix') || 'A folder named "',
|
||||||
|
folderNoteSuffix: t('install.folderNoteSuffix') || '" will be created',
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
106
apps/web/app/api/review/diagnose/route.ts
Normal file
106
apps/web/app/api/review/diagnose/route.ts
Normal file
@@ -0,0 +1,106 @@
|
|||||||
|
import { type NextRequest, NextResponse } from 'next/server';
|
||||||
|
import { createDb, skillQueries, sql } from '@skillhub/db';
|
||||||
|
import { requireAdmin } from '@/lib/admin-auth';
|
||||||
|
import { withRateLimit, createRateLimitResponse, createRateLimitHeaders } from '@/lib/rate-limit';
|
||||||
|
|
||||||
|
const db = createDb();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/review/diagnose?id=owner/repo/skill-name
|
||||||
|
* Returns which review pipeline filters a skill passes/fails.
|
||||||
|
* Calls the actual PostgreSQL raw_content_passes_prefilter function to detect
|
||||||
|
* discrepancies between JS approximation and SQL reality (e.g. invalid UTF-8).
|
||||||
|
* Admin-only endpoint for debugging why skills don't appear in pending list.
|
||||||
|
*/
|
||||||
|
export async function GET(request: NextRequest) {
|
||||||
|
const rateLimitResult = await withRateLimit(request, 'anonymous');
|
||||||
|
if (!rateLimitResult.allowed) {
|
||||||
|
return createRateLimitResponse(rateLimitResult);
|
||||||
|
}
|
||||||
|
|
||||||
|
const adminCheck = await requireAdmin(request);
|
||||||
|
if (!adminCheck.authorized) {
|
||||||
|
return adminCheck.response;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { searchParams } = new URL(request.url);
|
||||||
|
const skillId = searchParams.get('id');
|
||||||
|
|
||||||
|
if (!skillId) {
|
||||||
|
return NextResponse.json({ error: 'Missing id parameter' }, { status: 400 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const skill = await skillQueries.getById(db, skillId);
|
||||||
|
|
||||||
|
if (!skill) {
|
||||||
|
return NextResponse.json({ error: 'Skill not found', id: skillId }, { status: 404 });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Call the ACTUAL PostgreSQL function to check prefilter
|
||||||
|
// This catches UTF-8 issues that the JS approximation misses
|
||||||
|
let sqlPrefilterPass = false;
|
||||||
|
try {
|
||||||
|
const result = await db.execute(
|
||||||
|
sql`SELECT raw_content_passes_prefilter(raw_content) AS passes FROM skills WHERE id = ${skillId}`
|
||||||
|
);
|
||||||
|
const row = [...result][0] as { passes?: boolean } | undefined;
|
||||||
|
sqlPrefilterPass = row?.passes === true;
|
||||||
|
} catch {
|
||||||
|
sqlPrefilterPass = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// JS approximation for comparison
|
||||||
|
const rawContent = skill.rawContent ?? '';
|
||||||
|
const contentLength = Buffer.byteLength(rawContent, 'utf8');
|
||||||
|
const hasGeneratedComment = rawContent.includes('<!-- generated');
|
||||||
|
const hasUserPath = rawContent.substring(0, 1000).includes('/Users/') ||
|
||||||
|
rawContent.substring(0, 1000).includes('C:\\Users\\');
|
||||||
|
const jsPrefilterPass = contentLength >= 200 && !hasGeneratedComment && !hasUserPath;
|
||||||
|
|
||||||
|
const filters = {
|
||||||
|
// browseReadyFilter conditions
|
||||||
|
isDuplicate: { value: skill.isDuplicate, pass: !skill.isDuplicate || skill.isOwnerClaimed },
|
||||||
|
isStale: { value: skill.isStale, pass: !skill.isStale },
|
||||||
|
isMalicious: { value: skill.isMalicious, pass: !skill.isMalicious },
|
||||||
|
|
||||||
|
// Other conditions
|
||||||
|
isBlocked: { value: skill.isBlocked, pass: !skill.isBlocked },
|
||||||
|
sourceFormat: { value: skill.sourceFormat, pass: skill.sourceFormat === 'skill.md' },
|
||||||
|
isDeprecated: { value: skill.isDeprecated, pass: !skill.isDeprecated },
|
||||||
|
securityStatus: { value: skill.securityStatus, pass: skill.securityStatus === 'pass' },
|
||||||
|
qualityScore: { value: skill.qualityScore, pass: (skill.qualityScore ?? 0) >= 50 },
|
||||||
|
reviewStatus: { value: skill.reviewStatus, pass: skill.reviewStatus === 'auto-scored' },
|
||||||
|
|
||||||
|
// Prefilter: actual PostgreSQL function result
|
||||||
|
sqlPrefilter: { value: sqlPrefilterPass, pass: sqlPrefilterPass },
|
||||||
|
// JS approximation breakdown (for debugging discrepancies)
|
||||||
|
jsPrefilter: { value: jsPrefilterPass, pass: jsPrefilterPass },
|
||||||
|
contentLength: { value: contentLength, pass: contentLength >= 200 },
|
||||||
|
hasGeneratedComment: { value: hasGeneratedComment, pass: !hasGeneratedComment },
|
||||||
|
hasUserPath: { value: hasUserPath, pass: !hasUserPath },
|
||||||
|
};
|
||||||
|
|
||||||
|
// Use sqlPrefilter as the real filter (not JS approximation)
|
||||||
|
const failedFilters = Object.entries(filters)
|
||||||
|
.filter(([key, f]) => !f.pass && key !== 'jsPrefilter' && key !== 'contentLength' && key !== 'hasGeneratedComment' && key !== 'hasUserPath')
|
||||||
|
.map(([name]) => name);
|
||||||
|
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
id: skill.id,
|
||||||
|
name: skill.name,
|
||||||
|
downloadCount: skill.downloadCount,
|
||||||
|
wouldAppearInPending: failedFilters.length === 0,
|
||||||
|
failedFilters,
|
||||||
|
filters,
|
||||||
|
// Flag discrepancy between JS and SQL prefilter
|
||||||
|
...(jsPrefilterPass !== sqlPrefilterPass ? { prefilterDiscrepancy: true } : {}),
|
||||||
|
},
|
||||||
|
{ headers: createRateLimitHeaders(rateLimitResult) }
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
console.error('[Review] Diagnose error:', error);
|
||||||
|
return NextResponse.json({ error: 'Failed to diagnose skill' }, { status: 500 });
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -13,12 +13,16 @@ const db = createDb();
|
|||||||
* Supports owner-capped batches for diversity and hybrid re-review/new-review mixing.
|
* Supports owner-capped batches for diversity and hybrid re-review/new-review mixing.
|
||||||
*
|
*
|
||||||
* Query params:
|
* Query params:
|
||||||
* batch_size - number of skills to return (default 20, max 50)
|
* batch_size - number of skills to return (default 20, max 50)
|
||||||
* offset - number of skills to skip for pagination (default 0)
|
* offset - number of skills to skip for pagination (default 0)
|
||||||
* min_quality - minimum quality_score (default 50)
|
* min_quality - minimum quality_score (default 50)
|
||||||
* security - security_status filter (default "pass")
|
* security - security_status filter (default "pass")
|
||||||
* priority - "re-review" to show needs-re-review first, "re-review-all" to include already ai-reviewed skills
|
* priority - "re-review" to show needs-re-review first, "re-review-all" to include already ai-reviewed skills
|
||||||
* owner_limit - max skills per github_owner in batch (default 0=unlimited, max 10)
|
* owner_limit - max skills per github_owner in batch (default 0=unlimited, max 10)
|
||||||
|
* sort_by - sort order: "quality" (default), "stars", "downloads"
|
||||||
|
* min_ai_score - minimum latestAiScore filter (for targeted re-review)
|
||||||
|
* max_ai_score - maximum latestAiScore filter (for targeted re-review)
|
||||||
|
* reviewed_before - ISO date string, only include skills reviewed before this date
|
||||||
*/
|
*/
|
||||||
export async function GET(request: NextRequest) {
|
export async function GET(request: NextRequest) {
|
||||||
// Rate limiting
|
// Rate limiting
|
||||||
@@ -52,6 +56,14 @@ export async function GET(request: NextRequest) {
|
|||||||
const currentReviewVersion = Math.max(
|
const currentReviewVersion = Math.max(
|
||||||
parseInt(searchParams.get('review_version') ?? '0', 10) || 0, 0
|
parseInt(searchParams.get('review_version') ?? '0', 10) || 0, 0
|
||||||
);
|
);
|
||||||
|
const sortBy = (['quality', 'stars', 'downloads'] as const).includes(
|
||||||
|
searchParams.get('sort_by') as 'quality' | 'stars' | 'downloads'
|
||||||
|
) ? (searchParams.get('sort_by') as 'quality' | 'stars' | 'downloads') : 'quality';
|
||||||
|
const minAiScoreParam = searchParams.get('min_ai_score');
|
||||||
|
const minAiScore = minAiScoreParam ? parseInt(minAiScoreParam, 10) : undefined;
|
||||||
|
const maxAiScoreParam = searchParams.get('max_ai_score');
|
||||||
|
const maxAiScore = maxAiScoreParam ? parseInt(maxAiScoreParam, 10) : undefined;
|
||||||
|
const reviewedBefore = searchParams.get('reviewed_before') || undefined;
|
||||||
|
|
||||||
// Run counts in parallel
|
// Run counts in parallel
|
||||||
const [totalPending, reReviews] = await Promise.all([
|
const [totalPending, reReviews] = await Promise.all([
|
||||||
@@ -72,6 +84,10 @@ export async function GET(request: NextRequest) {
|
|||||||
reReviewAll: true,
|
reReviewAll: true,
|
||||||
ownerLimit,
|
ownerLimit,
|
||||||
currentReviewVersion,
|
currentReviewVersion,
|
||||||
|
sortBy,
|
||||||
|
minAiScore,
|
||||||
|
maxAiScore,
|
||||||
|
reviewedBefore,
|
||||||
}) as typeof batch;
|
}) as typeof batch;
|
||||||
batch = [...allBatch].slice(0, batchSize);
|
batch = [...allBatch].slice(0, batchSize);
|
||||||
} else {
|
} else {
|
||||||
@@ -86,6 +102,7 @@ export async function GET(request: NextRequest) {
|
|||||||
securityPass,
|
securityPass,
|
||||||
priorityReReview: true,
|
priorityReReview: true,
|
||||||
ownerLimit,
|
ownerLimit,
|
||||||
|
sortBy,
|
||||||
});
|
});
|
||||||
batch = [...reReviewBatch] as typeof batch;
|
batch = [...reReviewBatch] as typeof batch;
|
||||||
}
|
}
|
||||||
@@ -102,6 +119,7 @@ export async function GET(request: NextRequest) {
|
|||||||
securityPass,
|
securityPass,
|
||||||
priorityReReview,
|
priorityReReview,
|
||||||
ownerLimit,
|
ownerLimit,
|
||||||
|
sortBy,
|
||||||
}) as typeof batch;
|
}) as typeof batch;
|
||||||
|
|
||||||
if (priorityReReview) {
|
if (priorityReReview) {
|
||||||
|
|||||||
@@ -7,10 +7,8 @@ import { getCached, setCache, cacheKeys, cacheTTL } from '@/lib/cache';
|
|||||||
const db = createDb();
|
const db = createDb();
|
||||||
|
|
||||||
interface ReviewStatsData {
|
interface ReviewStatsData {
|
||||||
unreviewed: number;
|
total_skills: number;
|
||||||
auto_scored: number;
|
|
||||||
ai_reviewed: number;
|
ai_reviewed: number;
|
||||||
verified: number;
|
|
||||||
needs_re_review: number;
|
needs_re_review: number;
|
||||||
total_reviews: number;
|
total_reviews: number;
|
||||||
}
|
}
|
||||||
@@ -45,17 +43,18 @@ export async function GET(request: NextRequest) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run stats and total reviews count in parallel
|
// Run pipeline stats and total reviews count in parallel
|
||||||
const [statusCounts, totalReviews] = await Promise.all([
|
const [statusCounts, totalReviews] = await Promise.all([
|
||||||
skillReviewQueries.getStats(db),
|
skillReviewQueries.getPublicPipelineStats(db),
|
||||||
skillReviewQueries.countTotalReviews(db),
|
skillReviewQueries.countTotalReviews(db),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
// total_skills = sum of all statuses from pipeline query (browse-ready SKILL.md)
|
||||||
|
const totalSkills = Object.values(statusCounts).reduce((sum, n) => sum + n, 0);
|
||||||
|
|
||||||
const data: ReviewStatsData = {
|
const data: ReviewStatsData = {
|
||||||
unreviewed: statusCounts['unreviewed'] ?? 0,
|
total_skills: totalSkills,
|
||||||
auto_scored: statusCounts['auto-scored'] ?? 0,
|
|
||||||
ai_reviewed: statusCounts['ai-reviewed'] ?? 0,
|
ai_reviewed: statusCounts['ai-reviewed'] ?? 0,
|
||||||
verified: statusCounts['verified'] ?? 0,
|
|
||||||
needs_re_review: statusCounts['needs-re-review'] ?? 0,
|
needs_re_review: statusCounts['needs-re-review'] ?? 0,
|
||||||
total_reviews: totalReviews,
|
total_reviews: totalReviews,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { type NextRequest, NextResponse } from 'next/server';
|
import { type NextRequest, NextResponse } from 'next/server';
|
||||||
import { createDb, skillReviewQueries } from '@skillhub/db';
|
import { createDb, skillQueries, skillReviewQueries } from '@skillhub/db';
|
||||||
import { requireAdmin } from '@/lib/admin-auth';
|
import { requireAdmin } from '@/lib/admin-auth';
|
||||||
import { withRateLimit, createRateLimitResponse, createRateLimitHeaders } from '@/lib/rate-limit';
|
import { withRateLimit, createRateLimitResponse, createRateLimitHeaders } from '@/lib/rate-limit';
|
||||||
|
|
||||||
@@ -24,6 +24,7 @@ interface ReviewItem {
|
|||||||
set_verified?: boolean;
|
set_verified?: boolean;
|
||||||
review_version?: number;
|
review_version?: number;
|
||||||
reviewer?: string;
|
reviewer?: string;
|
||||||
|
recommendation?: 'flag-malicious' | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
function validateReviews(body: unknown): { reviews: ReviewItem[] } | { error: string } {
|
function validateReviews(body: unknown): { reviews: ReviewItem[] } | { error: string } {
|
||||||
@@ -74,6 +75,12 @@ function validateReviews(body: unknown): { reviews: ReviewItem[] } | { error: st
|
|||||||
return { error: `reviews[${i}].reviewer must be a non-empty string (max 50 chars)` };
|
return { error: `reviews[${i}].reviewer must be a non-empty string (max 50 chars)` };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Validate recommendation
|
||||||
|
if (item.recommendation !== undefined && item.recommendation !== null) {
|
||||||
|
if (item.recommendation !== 'flag-malicious') {
|
||||||
|
return { error: `reviews[${i}].recommendation must be 'flag-malicious' or null` };
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return { reviews: reviews as ReviewItem[] };
|
return { reviews: reviews as ReviewItem[] };
|
||||||
@@ -134,6 +141,7 @@ export async function POST(request: NextRequest) {
|
|||||||
i18nPriority: r.i18n_priority,
|
i18nPriority: r.i18n_priority,
|
||||||
contentHashAtReview: r.content_hash_at_review,
|
contentHashAtReview: r.content_hash_at_review,
|
||||||
reviewVersion: r.review_version,
|
reviewVersion: r.review_version,
|
||||||
|
recommendation: r.recommendation ?? undefined,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
await skillReviewQueries.createBatch(db, dbReviews);
|
await skillReviewQueries.createBatch(db, dbReviews);
|
||||||
@@ -147,10 +155,20 @@ export async function POST(request: NextRequest) {
|
|||||||
await skillReviewQueries.updateSkillReviewStatus(db, r.skill_id, newStatus, r.ai_score, new Date());
|
await skillReviewQueries.updateSkillReviewStatus(db, r.skill_id, newStatus, r.ai_score, new Date());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Flag malicious skills
|
||||||
|
let flaggedCount = 0;
|
||||||
|
for (const r of reviews) {
|
||||||
|
if (r.recommendation === 'flag-malicious') {
|
||||||
|
await skillQueries.flagMalicious(db, r.skill_id);
|
||||||
|
flaggedCount++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{
|
{
|
||||||
submitted: reviews.length,
|
submitted: reviews.length,
|
||||||
verified: verifiedCount,
|
verified: verifiedCount,
|
||||||
|
flagged: flaggedCount,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
headers: createRateLimitHeaders(rateLimitResult),
|
headers: createRateLimitHeaders(rateLimitResult),
|
||||||
|
|||||||
@@ -87,6 +87,14 @@ export async function GET(request: NextRequest) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Block file downloads for malicious skills
|
||||||
|
if (skill.isMalicious) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: 'This skill has been flagged as malicious. File downloads are blocked.', code: 'MALICIOUS' },
|
||||||
|
{ status: 403 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const { githubOwner, githubRepo, skillPath, branch, commitSha, sourceFormat } = skill;
|
const { githubOwner, githubRepo, skillPath, branch, commitSha, sourceFormat } = skill;
|
||||||
|
|
||||||
// === CACHE CHECK ===
|
// === CACHE CHECK ===
|
||||||
|
|||||||
@@ -84,8 +84,15 @@ export async function GET(
|
|||||||
downloadCount: skill.downloadCount,
|
downloadCount: skill.downloadCount,
|
||||||
viewCount: skill.viewCount,
|
viewCount: skill.viewCount,
|
||||||
securityScore: skill.securityScore,
|
securityScore: skill.securityScore,
|
||||||
|
securityStatus: skill.securityStatus,
|
||||||
|
qualityScore: skill.qualityScore,
|
||||||
isVerified: skill.isVerified,
|
isVerified: skill.isVerified,
|
||||||
isFeatured: skill.isFeatured,
|
isFeatured: skill.isFeatured,
|
||||||
|
isMalicious: skill.isMalicious ?? false,
|
||||||
|
isDuplicate: skill.isDuplicate ?? false,
|
||||||
|
isStale: skill.isStale ?? false,
|
||||||
|
isDeprecated: skill.isDeprecated ?? false,
|
||||||
|
isBlocked: skill.isBlocked ?? false,
|
||||||
compatibility: skill.compatibility,
|
compatibility: skill.compatibility,
|
||||||
triggers: skill.triggers,
|
triggers: skill.triggers,
|
||||||
rawContent: skill.rawContent,
|
rawContent: skill.rawContent,
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ export function Header() {
|
|||||||
{ name: t('home'), href: `/${locale}` },
|
{ name: t('home'), href: `/${locale}` },
|
||||||
{ name: t('browse'), href: `/${locale}/browse` },
|
{ name: t('browse'), href: `/${locale}/browse` },
|
||||||
{ name: t('categories'), href: `/${locale}/categories` },
|
{ name: t('categories'), href: `/${locale}/categories` },
|
||||||
|
{ name: t('reviewed'), href: `/${locale}/reviewed` },
|
||||||
{ name: t('docs'), href: `/${locale}/docs` },
|
{ name: t('docs'), href: `/${locale}/docs` },
|
||||||
];
|
];
|
||||||
|
|
||||||
|
|||||||
52
apps/web/components/SecurityAlertBanner.tsx
Normal file
52
apps/web/components/SecurityAlertBanner.tsx
Normal file
@@ -0,0 +1,52 @@
|
|||||||
|
'use client';
|
||||||
|
|
||||||
|
import { useState, useEffect } from 'react';
|
||||||
|
import { X, ShieldAlert } from 'lucide-react';
|
||||||
|
import { useTranslations } from 'next-intl';
|
||||||
|
|
||||||
|
const BLOG_POST_URL = 'https://blog.palebluedot.live/2026/03/19/malware-openclaw-skills-security-advisory/';
|
||||||
|
const STORAGE_KEY = 'security-alert-openclaw-dismissed';
|
||||||
|
|
||||||
|
export function SecurityAlertBanner() {
|
||||||
|
const [isVisible, setIsVisible] = useState(false);
|
||||||
|
const t = useTranslations('securityAlert');
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const dismissed = localStorage.getItem(STORAGE_KEY);
|
||||||
|
if (!dismissed) {
|
||||||
|
setIsVisible(true);
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const handleDismiss = () => {
|
||||||
|
setIsVisible(false);
|
||||||
|
localStorage.setItem(STORAGE_KEY, 'true');
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!isVisible) return null;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="relative bg-error-bg border-b-2 border-error">
|
||||||
|
<div className="container-main py-2.5 px-4 flex items-center justify-center gap-2 text-sm">
|
||||||
|
<ShieldAlert className="w-4 h-4 text-error flex-shrink-0" />
|
||||||
|
<span className="font-semibold text-error">{t('label')}</span>
|
||||||
|
<span className="text-text-primary">{t('text')}</span>
|
||||||
|
<a
|
||||||
|
href={BLOG_POST_URL}
|
||||||
|
target="_blank"
|
||||||
|
rel="noopener noreferrer"
|
||||||
|
className="text-error underline underline-offset-2 hover:text-error/80 transition-colors font-medium"
|
||||||
|
>
|
||||||
|
{t('link')}
|
||||||
|
</a>
|
||||||
|
<button
|
||||||
|
onClick={handleDismiss}
|
||||||
|
className="absolute end-2 sm:end-4 p-1 hover:bg-error/10 rounded transition-colors"
|
||||||
|
aria-label={t('dismiss')}
|
||||||
|
>
|
||||||
|
<X className="w-4 h-4 text-error" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
import Link from 'next/link';
|
import Link from 'next/link';
|
||||||
import { Star, Download, Shield, CheckCircle, Clock, RefreshCw, Sparkles } from 'lucide-react';
|
import { Star, Download, Shield, CheckCircle, Clock, RefreshCw, Sparkles, XCircle, ShieldAlert } from 'lucide-react';
|
||||||
import { formatCompactNumber } from '@/lib/format-number';
|
import { formatCompactNumber } from '@/lib/format-number';
|
||||||
|
|
||||||
const FORMAT_BADGE_LABELS: Record<string, string> = {
|
const FORMAT_BADGE_LABELS: Record<string, string> = {
|
||||||
@@ -27,6 +27,8 @@ interface SkillCardProps {
|
|||||||
sourceFormat?: string | null;
|
sourceFormat?: string | null;
|
||||||
createdAt?: Date | string | null;
|
createdAt?: Date | string | null;
|
||||||
updatedAt?: Date | string | null;
|
updatedAt?: Date | string | null;
|
||||||
|
reviewOutdated?: boolean | null;
|
||||||
|
isMalicious?: boolean | null;
|
||||||
};
|
};
|
||||||
locale: string;
|
locale: string;
|
||||||
/** Show time badge (for New Skills page) */
|
/** Show time badge (for New Skills page) */
|
||||||
@@ -56,16 +58,21 @@ export function SkillCard({ skill, locale, showTimeBadge, formatTimeAgo }: Skill
|
|||||||
|
|
||||||
const showRating = (skill.ratingCount ?? 0) >= 3;
|
const showRating = (skill.ratingCount ?? 0) >= 3;
|
||||||
|
|
||||||
// AI review score badge (only for reviewed skills with score >= 50)
|
// AI review score badge (for all reviewed skills)
|
||||||
const getAiScoreBadge = () => {
|
const getAiScoreBadge = () => {
|
||||||
const rs = skill.reviewStatus;
|
const rs = skill.reviewStatus;
|
||||||
if (!rs || rs === 'unreviewed' || rs === 'auto-scored') return null;
|
if (!rs || rs === 'unreviewed' || rs === 'auto-scored') return null;
|
||||||
const score = skill.aiScore ?? skill.latestAiScore;
|
const score = skill.aiScore ?? skill.latestAiScore;
|
||||||
if (!score || score < 50) return null;
|
if (score == null) return null;
|
||||||
|
if (score === 0) {
|
||||||
|
return { score: 0, color: 'text-error bg-error/10 border-error/20', rejected: true };
|
||||||
|
}
|
||||||
const color = score >= 75
|
const color = score >= 75
|
||||||
? 'text-success bg-success/10 border-success/20'
|
? 'text-success bg-success/10 border-success/20'
|
||||||
: 'text-gold bg-gold/10 border-gold/20';
|
: score >= 50
|
||||||
return { score, color };
|
? 'text-gold bg-gold/10 border-gold/20'
|
||||||
|
: 'text-text-muted bg-surface-subtle border-border';
|
||||||
|
return { score, color, rejected: false };
|
||||||
};
|
};
|
||||||
const aiScoreBadge = getAiScoreBadge();
|
const aiScoreBadge = getAiScoreBadge();
|
||||||
|
|
||||||
@@ -117,12 +124,28 @@ export function SkillCard({ skill, locale, showTimeBadge, formatTimeAgo }: Skill
|
|||||||
{skill.description}
|
{skill.description}
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
{/* Metadata Row: AI Score + Stars + Downloads + Rating */}
|
{/* Metadata Row: Malware / AI Score + Stars + Downloads + Rating */}
|
||||||
<div className="flex flex-wrap items-center gap-4 text-sm text-text-muted mb-2">
|
<div className="flex flex-wrap items-center gap-4 text-sm text-text-muted mb-2">
|
||||||
{aiScoreBadge && (
|
{skill.isMalicious && (
|
||||||
|
<span className="flex items-center gap-1 px-1.5 py-0.5 text-xs font-bold rounded border text-error bg-error/10 border-error/20">
|
||||||
|
<ShieldAlert className="w-3 h-3" />
|
||||||
|
<span>{locale === 'fa' ? 'بدافزار' : 'Malware'}</span>
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
{!skill.isMalicious && aiScoreBadge && (
|
||||||
<span className={`flex items-center gap-1 px-1.5 py-0.5 text-xs font-medium rounded border ${aiScoreBadge.color}`}>
|
<span className={`flex items-center gap-1 px-1.5 py-0.5 text-xs font-medium rounded border ${aiScoreBadge.color}`}>
|
||||||
<Sparkles className="w-3 h-3" />
|
{aiScoreBadge.rejected ? (
|
||||||
<span className="ltr-nums">{aiScoreBadge.score}</span>
|
<>
|
||||||
|
<XCircle className="w-3 h-3" />
|
||||||
|
<span>{locale === 'fa' ? 'رد شده' : 'Rejected'}</span>
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
<Sparkles className="w-3 h-3" />
|
||||||
|
<span className="ltr-nums">{aiScoreBadge.score}</span>
|
||||||
|
{skill.reviewOutdated && <span className="text-warning" title="Review based on previous version">⚠</span>}
|
||||||
|
</>
|
||||||
|
)}
|
||||||
</span>
|
</span>
|
||||||
)}
|
)}
|
||||||
<span className="flex items-center gap-1">
|
<span className="flex items-center gap-1">
|
||||||
|
|||||||
@@ -171,6 +171,8 @@ export const cacheKeys = {
|
|||||||
skillReview: (id: string) => `review:skill:${id.replace(/\//g, ':')}`,
|
skillReview: (id: string) => `review:skill:${id.replace(/\//g, ':')}`,
|
||||||
reviewStats: () => 'review:stats',
|
reviewStats: () => 'review:stats',
|
||||||
reviewedPage: (sort: string, page: number, minScore = 50) => `page:reviewed:${sort}:s${minScore}:${page}`,
|
reviewedPage: (sort: string, page: number, minScore = 50) => `page:reviewed:${sort}:s${minScore}:${page}`,
|
||||||
|
reviewStatsPublic: () => 'page:review-stats:public',
|
||||||
|
malwarePage: (page: number) => `page:malware:${page}`,
|
||||||
};
|
};
|
||||||
|
|
||||||
// TTL values in seconds
|
// TTL values in seconds
|
||||||
@@ -189,6 +191,8 @@ export const cacheTTL = {
|
|||||||
download: 5 * 60, // 5 minutes - same IP can only count as 1 download per 5 min
|
download: 5 * 60, // 5 minutes - same IP can only count as 1 download per 5 min
|
||||||
reviewStats: 60, // 1 minute - admin review stats
|
reviewStats: 60, // 1 minute - admin review stats
|
||||||
reviewed: 60 * 60, // 1 hour - reviewed skills page
|
reviewed: 60 * 60, // 1 hour - reviewed skills page
|
||||||
|
reviewStatsPublic: 30 * 60, // 30 minutes - public review stats page
|
||||||
|
malware: 60 * 60, // 1 hour - malware listing page
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -31,6 +31,7 @@
|
|||||||
"browse": "Browse",
|
"browse": "Browse",
|
||||||
"categories": "Categories",
|
"categories": "Categories",
|
||||||
"docs": "Docs",
|
"docs": "Docs",
|
||||||
|
"reviewed": "Reviewed",
|
||||||
"about": "About",
|
"about": "About",
|
||||||
"github": "GitHub",
|
"github": "GitHub",
|
||||||
"sourceCode": "Source Code",
|
"sourceCode": "Source Code",
|
||||||
@@ -464,7 +465,7 @@
|
|||||||
"platformWindsurf": "Windsurf — Windsurf config",
|
"platformWindsurf": "Windsurf — Windsurf config",
|
||||||
"fullDocsLink": "See the full discovery documentation for detailed prompts, search strategies, and advanced configuration.",
|
"fullDocsLink": "See the full discovery documentation for detailed prompts, search strategies, and advanced configuration.",
|
||||||
"cmdInstallDesc": "Install a skill (--platform, --project, --force, --no-api)",
|
"cmdInstallDesc": "Install a skill (--platform, --project, --force, --no-api)",
|
||||||
"cmdSearchDesc": "Search for skills (--platform, --limit, --page, --sort stars|downloads|rating|recent)",
|
"cmdSearchDesc": "Search for skills (--platform, --limit, --page, --sort recommended|aiScore|downloads|stars|rating|recent)",
|
||||||
"cmdListDesc": "List installed skills (--platform, --project, --all)",
|
"cmdListDesc": "List installed skills (--platform, --project, --all)",
|
||||||
"cmdUpdateDesc": "Update a skill or use --all to update all",
|
"cmdUpdateDesc": "Update a skill or use --all to update all",
|
||||||
"cmdUninstallDesc": "Remove an installed skill (--platform, --project)",
|
"cmdUninstallDesc": "Remove an installed skill (--platform, --project)",
|
||||||
@@ -472,7 +473,7 @@
|
|||||||
"exInstallGlobal": "Install globally:",
|
"exInstallGlobal": "Install globally:",
|
||||||
"exInstallProject": "Install in current project:",
|
"exInstallProject": "Install in current project:",
|
||||||
"exSearch": "Search for skills:",
|
"exSearch": "Search for skills:",
|
||||||
"exSearchSort": "Search sorted by stars:",
|
"exSearchSort": "Search sorted by AI review score:",
|
||||||
"exUpdateAll": "Update all installed skills:",
|
"exUpdateAll": "Update all installed skills:",
|
||||||
"platformsTitle": "Supported Platforms",
|
"platformsTitle": "Supported Platforms",
|
||||||
"platformsDesc": "SkillHub CLI supports 5 AI coding platforms:",
|
"platformsDesc": "SkillHub CLI supports 5 AI coding platforms:",
|
||||||
@@ -571,6 +572,44 @@
|
|||||||
"all": "All scores"
|
"all": "All scores"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"malicious": {
|
||||||
|
"warningTitle": "Malware Detected",
|
||||||
|
"warningDescription": "This skill has been flagged as malicious. It contains obfuscated code designed to download and execute harmful payloads. File downloads and installation are blocked.",
|
||||||
|
"installBlocked": "Installation blocked — this skill contains malware",
|
||||||
|
"badge": "Malware"
|
||||||
|
},
|
||||||
|
"reviewStats": {
|
||||||
|
"title": "Review Statistics",
|
||||||
|
"subtitle": "Overview of the AI review pipeline, score distribution, and security status",
|
||||||
|
"pipelineTitle": "Review Pipeline",
|
||||||
|
"pipelineDescription": "Status of skills in the review process",
|
||||||
|
"aiReviewed": "AI Reviewed",
|
||||||
|
"needsReReview": "Needs Re-review",
|
||||||
|
"totalReviews": "Total Review Records",
|
||||||
|
"scoreTitle": "Score Distribution",
|
||||||
|
"scoreDescription": "Distribution of AI review scores across reviewed skills",
|
||||||
|
"scoreHigh": "High Quality (75–100)",
|
||||||
|
"scoreMid": "Solid (50–74)",
|
||||||
|
"scoreLow": "Below Average (0–49)",
|
||||||
|
"securityTitle": "Security Scan",
|
||||||
|
"securityDescription": "Automated security scan results for all skills",
|
||||||
|
"securityPass": "Pass",
|
||||||
|
"securityWarning": "Warning",
|
||||||
|
"securityFail": "Fail",
|
||||||
|
"malwareTitle": "Malware Detection",
|
||||||
|
"malwareDescription": "Skills flagged as containing malicious code",
|
||||||
|
"malwareFlagged": "Flagged as Malware",
|
||||||
|
"malwareViewAll": "View flagged skills",
|
||||||
|
"skills": "skills"
|
||||||
|
},
|
||||||
|
"malwarePage": {
|
||||||
|
"title": "Flagged Malware Skills",
|
||||||
|
"subtitle": "These skills have been identified as containing malicious code. Downloads and installation are blocked for your safety.",
|
||||||
|
"noResults": "No malware-flagged skills found.",
|
||||||
|
"flaggedOn": "Flagged on",
|
||||||
|
"downloadBlocked": "Downloads blocked",
|
||||||
|
"installBlocked": "Installation blocked"
|
||||||
|
},
|
||||||
"owner": {
|
"owner": {
|
||||||
"title": "@{username}'s Skills",
|
"title": "@{username}'s Skills",
|
||||||
"notFound": "Owner not found",
|
"notFound": "Owner not found",
|
||||||
@@ -933,6 +972,12 @@
|
|||||||
"feedback": "Feedback",
|
"feedback": "Feedback",
|
||||||
"dismiss": "Dismiss"
|
"dismiss": "Dismiss"
|
||||||
},
|
},
|
||||||
|
"securityAlert": {
|
||||||
|
"label": "Security Alert:",
|
||||||
|
"text": "Malware found in 5 skills from openclaw/skills.",
|
||||||
|
"link": "Read the advisory",
|
||||||
|
"dismiss": "Dismiss"
|
||||||
|
},
|
||||||
"email": {
|
"email": {
|
||||||
"welcome": {
|
"welcome": {
|
||||||
"subject": "Welcome to SkillHub!",
|
"subject": "Welcome to SkillHub!",
|
||||||
|
|||||||
@@ -31,6 +31,7 @@
|
|||||||
"browse": "مرور",
|
"browse": "مرور",
|
||||||
"categories": "دستهبندیها",
|
"categories": "دستهبندیها",
|
||||||
"docs": "مستندات",
|
"docs": "مستندات",
|
||||||
|
"reviewed": "بررسیشده",
|
||||||
"about": "درباره ما",
|
"about": "درباره ما",
|
||||||
"github": "گیتهاب",
|
"github": "گیتهاب",
|
||||||
"sourceCode": "کد منبع",
|
"sourceCode": "کد منبع",
|
||||||
@@ -464,7 +465,7 @@
|
|||||||
"platformWindsurf": "Windsurf — پیکربندی Windsurf",
|
"platformWindsurf": "Windsurf — پیکربندی Windsurf",
|
||||||
"fullDocsLink": "مستندات کامل کشف را برای پرامپتهای دقیق، استراتژیهای جستجو و پیکربندی پیشرفته ببینید.",
|
"fullDocsLink": "مستندات کامل کشف را برای پرامپتهای دقیق، استراتژیهای جستجو و پیکربندی پیشرفته ببینید.",
|
||||||
"cmdInstallDesc": "نصب مهارت (--platform, --project, --force, --no-api)",
|
"cmdInstallDesc": "نصب مهارت (--platform, --project, --force, --no-api)",
|
||||||
"cmdSearchDesc": "جستجوی مهارتها (--platform, --limit, --page, --sort stars|downloads|rating|recent)",
|
"cmdSearchDesc": "جستجوی مهارتها (--platform, --limit, --page, --sort recommended|aiScore|downloads|stars|rating|recent)",
|
||||||
"cmdListDesc": "لیست مهارتهای نصبشده (--platform, --project, --all)",
|
"cmdListDesc": "لیست مهارتهای نصبشده (--platform, --project, --all)",
|
||||||
"cmdUpdateDesc": "بهروزرسانی مهارت یا استفاده از --all برای بهروزرسانی همه",
|
"cmdUpdateDesc": "بهروزرسانی مهارت یا استفاده از --all برای بهروزرسانی همه",
|
||||||
"cmdUninstallDesc": "حذف مهارت نصبشده (--platform, --project)",
|
"cmdUninstallDesc": "حذف مهارت نصبشده (--platform, --project)",
|
||||||
@@ -472,7 +473,7 @@
|
|||||||
"exInstallGlobal": "نصب سراسری:",
|
"exInstallGlobal": "نصب سراسری:",
|
||||||
"exInstallProject": "نصب در پروژه فعلی:",
|
"exInstallProject": "نصب در پروژه فعلی:",
|
||||||
"exSearch": "جستجوی مهارتها:",
|
"exSearch": "جستجوی مهارتها:",
|
||||||
"exSearchSort": "جستجو بر اساس ستارهها:",
|
"exSearchSort": "جستجو بر اساس امتیاز بررسی:",
|
||||||
"exUpdateAll": "بهروزرسانی همه مهارتها:",
|
"exUpdateAll": "بهروزرسانی همه مهارتها:",
|
||||||
"platformsTitle": "پلتفرمهای پشتیبانیشده",
|
"platformsTitle": "پلتفرمهای پشتیبانیشده",
|
||||||
"platformsDesc": "CLI SkillHub از ۵ پلتفرم کدنویسی هوش مصنوعی پشتیبانی میکند:",
|
"platformsDesc": "CLI SkillHub از ۵ پلتفرم کدنویسی هوش مصنوعی پشتیبانی میکند:",
|
||||||
@@ -571,6 +572,44 @@
|
|||||||
"all": "همه امتیازها"
|
"all": "همه امتیازها"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"malicious": {
|
||||||
|
"warningTitle": "بدافزار شناسایی شد",
|
||||||
|
"warningDescription": "این مهارت به عنوان مخرب شناسایی شده است. شامل کد مبهمسازی شده برای دانلود و اجرای بارهای مضر است. دانلود فایل و نصب مسدود شده است.",
|
||||||
|
"installBlocked": "نصب مسدود شده — این مهارت حاوی بدافزار است",
|
||||||
|
"badge": "بدافزار"
|
||||||
|
},
|
||||||
|
"reviewStats": {
|
||||||
|
"title": "آمار بررسیها",
|
||||||
|
"subtitle": "نمای کلی از خط لوله بررسی هوش مصنوعی، توزیع امتیاز و وضعیت امنیتی",
|
||||||
|
"pipelineTitle": "خط لوله بررسی",
|
||||||
|
"pipelineDescription": "وضعیت مهارتها در فرآیند بررسی",
|
||||||
|
"aiReviewed": "بررسیشده توسط AI",
|
||||||
|
"needsReReview": "نیاز به بررسی مجدد",
|
||||||
|
"totalReviews": "کل رکوردهای بررسی",
|
||||||
|
"scoreTitle": "توزیع امتیاز",
|
||||||
|
"scoreDescription": "توزیع امتیازهای بررسی هوش مصنوعی در مهارتهای بررسیشده",
|
||||||
|
"scoreHigh": "کیفیت بالا (۷۵–۱۰۰)",
|
||||||
|
"scoreMid": "قابل قبول (۵۰–۷۴)",
|
||||||
|
"scoreLow": "زیر متوسط (۰–۴۹)",
|
||||||
|
"securityTitle": "اسکن امنیتی",
|
||||||
|
"securityDescription": "نتایج اسکن امنیتی خودکار برای تمام مهارتها",
|
||||||
|
"securityPass": "تایید",
|
||||||
|
"securityWarning": "هشدار",
|
||||||
|
"securityFail": "رد",
|
||||||
|
"malwareTitle": "شناسایی بدافزار",
|
||||||
|
"malwareDescription": "مهارتهایی که حاوی کد مخرب شناسایی شدهاند",
|
||||||
|
"malwareFlagged": "فلگ شده به عنوان بدافزار",
|
||||||
|
"malwareViewAll": "مشاهده مهارتهای فلگشده",
|
||||||
|
"skills": "مهارت"
|
||||||
|
},
|
||||||
|
"malwarePage": {
|
||||||
|
"title": "مهارتهای فلگشده به عنوان بدافزار",
|
||||||
|
"subtitle": "این مهارتها حاوی کد مخرب شناسایی شدهاند. دانلود و نصب آنها برای امنیت شما مسدود شده است.",
|
||||||
|
"noResults": "هیچ مهارت بدافزاردار فلگشدهای یافت نشد.",
|
||||||
|
"flaggedOn": "فلگ شده در",
|
||||||
|
"downloadBlocked": "دانلود مسدود شده",
|
||||||
|
"installBlocked": "نصب مسدود شده"
|
||||||
|
},
|
||||||
"owner": {
|
"owner": {
|
||||||
"title": "مهارتهای @{username}",
|
"title": "مهارتهای @{username}",
|
||||||
"notFound": "مالک یافت نشد",
|
"notFound": "مالک یافت نشد",
|
||||||
@@ -933,6 +972,12 @@
|
|||||||
"feedback": "بازخورد",
|
"feedback": "بازخورد",
|
||||||
"dismiss": "بستن"
|
"dismiss": "بستن"
|
||||||
},
|
},
|
||||||
|
"securityAlert": {
|
||||||
|
"label": "هشدار امنیتی:",
|
||||||
|
"text": "بدافزار در ۵ مهارت از openclaw/skills شناسایی شد.",
|
||||||
|
"link": "مشاهده اطلاعیه",
|
||||||
|
"dismiss": "بستن"
|
||||||
|
},
|
||||||
"email": {
|
"email": {
|
||||||
"welcome": {
|
"welcome": {
|
||||||
"subject": "به SkillHub خوش آمدید!",
|
"subject": "به SkillHub خوش آمدید!",
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ type DB = PostgresJsDatabase<typeof schema>;
|
|||||||
* Skills with skill_type=NULL (newly crawled, not yet curated) are included
|
* Skills with skill_type=NULL (newly crawled, not yet curated) are included
|
||||||
* so they don't disappear between curate.mjs runs.
|
* so they don't disappear between curate.mjs runs.
|
||||||
*/
|
*/
|
||||||
const browseReadyFilter = sql`(${skills.isDuplicate} = false OR ${skills.isOwnerClaimed} = true) AND ${skills.isStale} = false`;
|
const browseReadyFilter = sql`(${skills.isDuplicate} = false OR ${skills.isOwnerClaimed} = true) AND ${skills.isStale} = false AND ${skills.isMalicious} = false`;
|
||||||
|
|
||||||
/** Minimum repo age gap (in days) to trust repo_created_at over stars for duplicate detection */
|
/** Minimum repo age gap (in days) to trust repo_created_at over stars for duplicate detection */
|
||||||
const REPO_AGE_THRESHOLD_DAYS = 75;
|
const REPO_AGE_THRESHOLD_DAYS = 75;
|
||||||
@@ -483,11 +483,41 @@ export const skillQueries = {
|
|||||||
if (minScore > 0) {
|
if (minScore > 0) {
|
||||||
conditions.push(sql`${skills.latestAiScore} >= ${minScore}`);
|
conditions.push(sql`${skills.latestAiScore} >= ${minScore}`);
|
||||||
}
|
}
|
||||||
return db.select().from(skills)
|
const results = await db.select().from(skills)
|
||||||
.where(and(...conditions))
|
.where(and(...conditions))
|
||||||
.orderBy(order)
|
.orderBy(order)
|
||||||
.limit(limit)
|
.limit(limit)
|
||||||
.offset(offset);
|
.offset(offset);
|
||||||
|
|
||||||
|
if (results.length === 0) return results.map(s => ({ ...s, reviewOutdated: false as boolean }));
|
||||||
|
|
||||||
|
// Batch-fetch latest review hashes for these skills
|
||||||
|
const ids = results.map(s => s.id);
|
||||||
|
const reviewRows = await db
|
||||||
|
.select({
|
||||||
|
skillId: skillReviews.skillId,
|
||||||
|
hashAtReview: skillReviews.contentHashAtReview,
|
||||||
|
})
|
||||||
|
.from(skillReviews)
|
||||||
|
.where(and(
|
||||||
|
inArray(skillReviews.skillId, ids),
|
||||||
|
sql`${skillReviews.contentHashAtReview} IS NOT NULL AND length(${skillReviews.contentHashAtReview}) > 1`,
|
||||||
|
))
|
||||||
|
.orderBy(desc(skillReviews.reviewedAt));
|
||||||
|
// Keep only the latest review per skill
|
||||||
|
const reviewHashMap = new Map<string, string>();
|
||||||
|
for (const r of reviewRows) {
|
||||||
|
if (!reviewHashMap.has(r.skillId) && r.hashAtReview && r.hashAtReview.length > 1) {
|
||||||
|
reviewHashMap.set(r.skillId, r.hashAtReview);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return results.map(s => ({
|
||||||
|
...s,
|
||||||
|
reviewOutdated: reviewHashMap.has(s.id)
|
||||||
|
&& s.contentHash != null && s.contentHash.length > 1
|
||||||
|
&& reviewHashMap.get(s.id) !== s.contentHash,
|
||||||
|
}));
|
||||||
},
|
},
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -760,7 +790,7 @@ export const skillQueries = {
|
|||||||
await db
|
await db
|
||||||
.update(skills)
|
.update(skills)
|
||||||
.set({
|
.set({
|
||||||
downloadCount: sql`${skills.downloadCount} + 1`,
|
downloadCount: sql`COALESCE(${skills.downloadCount}, 0) + 1`,
|
||||||
lastDownloadedAt: new Date(),
|
lastDownloadedAt: new Date(),
|
||||||
})
|
})
|
||||||
.where(eq(skills.id, id));
|
.where(eq(skills.id, id));
|
||||||
@@ -817,6 +847,23 @@ export const skillQueries = {
|
|||||||
await db.update(skills).set({ isBlocked: false }).where(eq(skills.id, id));
|
await db.update(skills).set({ isBlocked: false }).where(eq(skills.id, id));
|
||||||
},
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Flag a skill as malicious (contains malware)
|
||||||
|
*/
|
||||||
|
flagMalicious: async (db: DB, id: string) => {
|
||||||
|
await db.update(skills).set({
|
||||||
|
isMalicious: true,
|
||||||
|
securityStatus: 'fail' as const,
|
||||||
|
}).where(eq(skills.id, id));
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Unflag a skill as malicious
|
||||||
|
*/
|
||||||
|
unflagMalicious: async (db: DB, id: string) => {
|
||||||
|
await db.update(skills).set({ isMalicious: false }).where(eq(skills.id, id));
|
||||||
|
},
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Mark all skills from a repo as owner-claimed (exempt from duplicate hiding)
|
* Mark all skills from a repo as owner-claimed (exempt from duplicate hiding)
|
||||||
*/
|
*/
|
||||||
@@ -2528,6 +2575,7 @@ export const skillReviewQueries = {
|
|||||||
i18nPriority?: number;
|
i18nPriority?: number;
|
||||||
contentHashAtReview?: string;
|
contentHashAtReview?: string;
|
||||||
reviewVersion?: number;
|
reviewVersion?: number;
|
||||||
|
recommendation?: string;
|
||||||
}>
|
}>
|
||||||
) => {
|
) => {
|
||||||
return db
|
return db
|
||||||
@@ -2549,6 +2597,7 @@ export const skillReviewQueries = {
|
|||||||
i18nPriority: r.i18nPriority ?? 0,
|
i18nPriority: r.i18nPriority ?? 0,
|
||||||
contentHashAtReview: r.contentHashAtReview,
|
contentHashAtReview: r.contentHashAtReview,
|
||||||
reviewVersion: r.reviewVersion ?? 1,
|
reviewVersion: r.reviewVersion ?? 1,
|
||||||
|
recommendation: r.recommendation,
|
||||||
}))
|
}))
|
||||||
)
|
)
|
||||||
.returning();
|
.returning();
|
||||||
@@ -2693,6 +2742,117 @@ export const skillReviewQueries = {
|
|||||||
return result[0]?.count ?? 0;
|
return result[0]?.count ?? 0;
|
||||||
},
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get pipeline stats for browse-ready SKILL.md skills (matches site-visible skills)
|
||||||
|
*/
|
||||||
|
getPublicPipelineStats: async (db: DB) => {
|
||||||
|
const result = await db
|
||||||
|
.select({
|
||||||
|
reviewStatus: skills.reviewStatus,
|
||||||
|
count: sql<number>`count(*)::int`,
|
||||||
|
})
|
||||||
|
.from(skills)
|
||||||
|
.where(
|
||||||
|
and(
|
||||||
|
eq(skills.isBlocked, false),
|
||||||
|
eq(skills.sourceFormat, 'skill.md'),
|
||||||
|
browseReadyFilter,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.groupBy(skills.reviewStatus);
|
||||||
|
|
||||||
|
const stats: Record<string, number> = {};
|
||||||
|
for (const row of result) {
|
||||||
|
stats[row.reviewStatus ?? 'unreviewed'] = row.count;
|
||||||
|
}
|
||||||
|
return stats;
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get AI score distribution for reviewed skills (matches badge system: 75+, 50-74, <50)
|
||||||
|
*/
|
||||||
|
getScoreDistribution: async (db: DB) => {
|
||||||
|
const result = await db
|
||||||
|
.select({
|
||||||
|
range: sql<string>`CASE
|
||||||
|
WHEN ${skills.latestAiScore} >= 75 THEN 'high'
|
||||||
|
WHEN ${skills.latestAiScore} >= 50 THEN 'mid'
|
||||||
|
ELSE 'low'
|
||||||
|
END`,
|
||||||
|
count: sql<number>`count(*)::int`,
|
||||||
|
})
|
||||||
|
.from(skills)
|
||||||
|
.where(
|
||||||
|
and(
|
||||||
|
eq(skills.isBlocked, false),
|
||||||
|
eq(skills.sourceFormat, 'skill.md'),
|
||||||
|
browseReadyFilter,
|
||||||
|
inArray(skills.reviewStatus, ['ai-reviewed', 'verified']),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.groupBy(sql`CASE
|
||||||
|
WHEN ${skills.latestAiScore} >= 75 THEN 'high'
|
||||||
|
WHEN ${skills.latestAiScore} >= 50 THEN 'mid'
|
||||||
|
ELSE 'low'
|
||||||
|
END`);
|
||||||
|
|
||||||
|
const dist: Record<string, number> = { high: 0, mid: 0, low: 0 };
|
||||||
|
for (const row of result) {
|
||||||
|
dist[row.range] = row.count;
|
||||||
|
}
|
||||||
|
return dist;
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get security scan stats (counts by security_status)
|
||||||
|
*/
|
||||||
|
getSecurityStats: async (db: DB) => {
|
||||||
|
const result = await db
|
||||||
|
.select({
|
||||||
|
status: skills.securityStatus,
|
||||||
|
count: sql<number>`count(*)::int`,
|
||||||
|
})
|
||||||
|
.from(skills)
|
||||||
|
.where(
|
||||||
|
and(
|
||||||
|
eq(skills.isBlocked, false),
|
||||||
|
eq(skills.sourceFormat, 'skill.md'),
|
||||||
|
sql`${skills.securityStatus} IS NOT NULL`,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.groupBy(skills.securityStatus);
|
||||||
|
|
||||||
|
const stats: Record<string, number> = { pass: 0, warning: 0, fail: 0 };
|
||||||
|
for (const row of result) {
|
||||||
|
if (row.status) stats[row.status] = row.count;
|
||||||
|
}
|
||||||
|
return stats;
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get malware-flagged skills (for public malware listing page)
|
||||||
|
*/
|
||||||
|
getMaliciousSkills: async (db: DB, limit = 50, offset = 0) => {
|
||||||
|
return db
|
||||||
|
.select()
|
||||||
|
.from(skills)
|
||||||
|
.where(and(eq(skills.isMalicious, true), eq(skills.isBlocked, false)))
|
||||||
|
.orderBy(desc(skills.updatedAt))
|
||||||
|
.limit(limit)
|
||||||
|
.offset(offset);
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Count malware-flagged skills
|
||||||
|
*/
|
||||||
|
countMaliciousSkills: async (db: DB) => {
|
||||||
|
const result = await db
|
||||||
|
.select({ count: sql<number>`count(*)::int` })
|
||||||
|
.from(skills)
|
||||||
|
.where(and(eq(skills.isMalicious, true), eq(skills.isBlocked, false)));
|
||||||
|
return result[0]?.count ?? 0;
|
||||||
|
},
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get skills pending AI review (auto-scored, security=pass, quality>=threshold)
|
* Get skills pending AI review (auto-scored, security=pass, quality>=threshold)
|
||||||
* Only returns standard SKILL.md format skills (excludes agents.md, cursorrules, etc.)
|
* Only returns standard SKILL.md format skills (excludes agents.md, cursorrules, etc.)
|
||||||
@@ -2710,6 +2870,10 @@ export const skillReviewQueries = {
|
|||||||
reReviewAll?: boolean;
|
reReviewAll?: boolean;
|
||||||
ownerLimit?: number;
|
ownerLimit?: number;
|
||||||
currentReviewVersion?: number;
|
currentReviewVersion?: number;
|
||||||
|
sortBy?: 'quality' | 'stars' | 'downloads';
|
||||||
|
minAiScore?: number;
|
||||||
|
maxAiScore?: number;
|
||||||
|
reviewedBefore?: string;
|
||||||
} = {}
|
} = {}
|
||||||
) => {
|
) => {
|
||||||
const {
|
const {
|
||||||
@@ -2721,6 +2885,10 @@ export const skillReviewQueries = {
|
|||||||
reReviewAll = false,
|
reReviewAll = false,
|
||||||
ownerLimit = 0,
|
ownerLimit = 0,
|
||||||
currentReviewVersion = 0,
|
currentReviewVersion = 0,
|
||||||
|
sortBy = 'quality',
|
||||||
|
minAiScore,
|
||||||
|
maxAiScore,
|
||||||
|
reviewedBefore,
|
||||||
} = options;
|
} = options;
|
||||||
|
|
||||||
const conditions = [
|
const conditions = [
|
||||||
@@ -2763,6 +2931,19 @@ export const skillReviewQueries = {
|
|||||||
conditions.push(eq(skills.securityStatus, 'pass'));
|
conditions.push(eq(skills.securityStatus, 'pass'));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AI score range filters (for targeted re-review of inflated scores)
|
||||||
|
if (minAiScore != null) {
|
||||||
|
conditions.push(gte(skills.latestAiScore, minAiScore));
|
||||||
|
}
|
||||||
|
if (maxAiScore != null) {
|
||||||
|
conditions.push(sql`${skills.latestAiScore} <= ${maxAiScore}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Date filter: only include skills reviewed before a given date
|
||||||
|
if (reviewedBefore) {
|
||||||
|
conditions.push(sql`${skills.latestReviewDate} < ${reviewedBefore}::timestamp`);
|
||||||
|
}
|
||||||
|
|
||||||
const selectFields = {
|
const selectFields = {
|
||||||
id: skills.id,
|
id: skills.id,
|
||||||
name: skills.name,
|
name: skills.name,
|
||||||
@@ -2771,6 +2952,7 @@ export const skillReviewQueries = {
|
|||||||
qualityScore: skills.qualityScore,
|
qualityScore: skills.qualityScore,
|
||||||
securityStatus: skills.securityStatus,
|
securityStatus: skills.securityStatus,
|
||||||
githubStars: skills.githubStars,
|
githubStars: skills.githubStars,
|
||||||
|
downloadCount: skills.downloadCount,
|
||||||
skillType: skills.skillType,
|
skillType: skills.skillType,
|
||||||
reviewStatus: skills.reviewStatus,
|
reviewStatus: skills.reviewStatus,
|
||||||
contentHash: skills.contentHash,
|
contentHash: skills.contentHash,
|
||||||
@@ -2780,11 +2962,21 @@ export const skillReviewQueries = {
|
|||||||
branch: skills.branch,
|
branch: skills.branch,
|
||||||
};
|
};
|
||||||
|
|
||||||
// Re-review-all: prioritize already-reviewed skills first, then auto-scored
|
// Build sort expression based on sortBy option
|
||||||
// Use sql.raw() for the ORDER BY to avoid parameterization issues in db.execute()
|
// Use COALESCE to handle NULL download_count/quality_score (NULL + 1 = NULL means
|
||||||
const orderBySql = reReviewAll
|
// skills never downloaded have NULL, not 0). Use sql.raw() for ORDER BY to avoid
|
||||||
? sql`CASE WHEN review_status IN ('ai-reviewed', 'needs-re-review') THEN 0 ELSE 1 END, quality_score DESC NULLS LAST, github_stars DESC NULLS LAST`
|
// Drizzle parameterizing the fragment (which turns it into a string constant).
|
||||||
: sql`quality_score DESC NULLS LAST, github_stars DESC NULLS LAST`;
|
const sortExprRaw = sortBy === 'downloads'
|
||||||
|
? `COALESCE(download_count, 0) DESC, COALESCE(quality_score, 0) DESC`
|
||||||
|
: sortBy === 'stars'
|
||||||
|
? `COALESCE(github_stars, 0) DESC, COALESCE(quality_score, 0) DESC`
|
||||||
|
: `COALESCE(quality_score, 0) DESC, COALESCE(github_stars, 0) DESC`;
|
||||||
|
|
||||||
|
// Re-review-all: prioritize already-reviewed skills first, then apply sort
|
||||||
|
const orderByRaw = reReviewAll
|
||||||
|
? `CASE WHEN review_status IN ('ai-reviewed', 'needs-re-review') THEN 0 ELSE 1 END, ${sortExprRaw}`
|
||||||
|
: sortExprRaw;
|
||||||
|
const orderBySql = sql.raw(orderByRaw);
|
||||||
|
|
||||||
// Owner-capped batch: limit skills per github_owner for diversity
|
// Owner-capped batch: limit skills per github_owner for diversity
|
||||||
if (ownerLimit > 0) {
|
if (ownerLimit > 0) {
|
||||||
@@ -2794,6 +2986,7 @@ export const skillReviewQueries = {
|
|||||||
SELECT id, name, description, raw_content, quality_score, security_status,
|
SELECT id, name, description, raw_content, quality_score, security_status,
|
||||||
github_stars, skill_type, review_status, content_hash,
|
github_stars, skill_type, review_status, content_hash,
|
||||||
github_owner, github_repo, skill_path, branch,
|
github_owner, github_repo, skill_path, branch,
|
||||||
|
download_count, latest_ai_score, latest_review_date,
|
||||||
ROW_NUMBER() OVER (
|
ROW_NUMBER() OVER (
|
||||||
PARTITION BY github_owner
|
PARTITION BY github_owner
|
||||||
ORDER BY ${orderBySql}
|
ORDER BY ${orderBySql}
|
||||||
@@ -2803,7 +2996,8 @@ export const skillReviewQueries = {
|
|||||||
)
|
)
|
||||||
SELECT id, name, description, raw_content AS "rawContent",
|
SELECT id, name, description, raw_content AS "rawContent",
|
||||||
quality_score AS "qualityScore", security_status AS "securityStatus",
|
quality_score AS "qualityScore", security_status AS "securityStatus",
|
||||||
github_stars AS "githubStars", skill_type AS "skillType",
|
github_stars AS "githubStars", download_count AS "downloadCount",
|
||||||
|
skill_type AS "skillType",
|
||||||
review_status AS "reviewStatus", content_hash AS "contentHash",
|
review_status AS "reviewStatus", content_hash AS "contentHash",
|
||||||
github_owner AS "githubOwner", github_repo AS "githubRepo",
|
github_owner AS "githubOwner", github_repo AS "githubRepo",
|
||||||
skill_path AS "skillPath", branch
|
skill_path AS "skillPath", branch
|
||||||
@@ -2816,10 +3010,18 @@ export const skillReviewQueries = {
|
|||||||
return [...results];
|
return [...results];
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build order: re-review priority first (if applicable), then quality
|
// Build order: re-review priority first (if applicable), then sort
|
||||||
|
// Use COALESCE to handle NULLs (download_count is NULL for never-downloaded skills,
|
||||||
|
// not 0, because increment uses NULL + 1 = NULL). Without COALESCE, NULLs sort
|
||||||
|
// first in DESC order, making 50K+ NULL skills appear before high-download skills.
|
||||||
|
const baseSortClauses = sortBy === 'downloads'
|
||||||
|
? [sql`COALESCE(${skills.downloadCount}, 0) DESC`, sql`COALESCE(${skills.qualityScore}, 0) DESC`]
|
||||||
|
: sortBy === 'stars'
|
||||||
|
? [sql`COALESCE(${skills.githubStars}, 0) DESC`, sql`COALESCE(${skills.qualityScore}, 0) DESC`]
|
||||||
|
: [sql`COALESCE(${skills.qualityScore}, 0) DESC`, sql`COALESCE(${skills.githubStars}, 0) DESC`];
|
||||||
const orderClauses = reReviewAll
|
const orderClauses = reReviewAll
|
||||||
? [sql`CASE WHEN ${skills.reviewStatus} IN ('ai-reviewed', 'needs-re-review') THEN 0 ELSE 1 END`, desc(skills.qualityScore), desc(skills.githubStars)]
|
? [sql`CASE WHEN ${skills.reviewStatus} IN ('ai-reviewed', 'needs-re-review') THEN 0 ELSE 1 END`, ...baseSortClauses]
|
||||||
: [desc(skills.qualityScore), desc(skills.githubStars)];
|
: baseSortClauses;
|
||||||
|
|
||||||
return db
|
return db
|
||||||
.select(selectFields)
|
.select(selectFields)
|
||||||
|
|||||||
@@ -66,6 +66,7 @@ export const skills = pgTable(
|
|||||||
isVerified: boolean('is_verified').default(false),
|
isVerified: boolean('is_verified').default(false),
|
||||||
isFeatured: boolean('is_featured').default(false),
|
isFeatured: boolean('is_featured').default(false),
|
||||||
isBlocked: boolean('is_blocked').default(false), // Blocked from re-indexing (owner requested removal)
|
isBlocked: boolean('is_blocked').default(false), // Blocked from re-indexing (owner requested removal)
|
||||||
|
isMalicious: boolean('is_malicious').default(false), // Flagged as containing malware/malicious code
|
||||||
isDeprecated: boolean('is_deprecated').default(false), // Auto-detected from raw_content (DEPRECATED/ARCHIVED markers)
|
isDeprecated: boolean('is_deprecated').default(false), // Auto-detected from raw_content (DEPRECATED/ARCHIVED markers)
|
||||||
isStale: boolean('is_stale').default(false), // Skill files no longer accessible on GitHub (confirmed after 3 consecutive 404s)
|
isStale: boolean('is_stale').default(false), // Skill files no longer accessible on GitHub (confirmed after 3 consecutive 404s)
|
||||||
staleSince: timestamp('stale_since'), // When staleness was first detected
|
staleSince: timestamp('stale_since'), // When staleness was first detected
|
||||||
@@ -431,6 +432,7 @@ export const skillReviews = pgTable(
|
|||||||
contentHashAtReview: text('content_hash_at_review'), // skill hash at time of review
|
contentHashAtReview: text('content_hash_at_review'), // skill hash at time of review
|
||||||
reviewedAt: timestamp('reviewed_at').defaultNow(),
|
reviewedAt: timestamp('reviewed_at').defaultNow(),
|
||||||
reviewVersion: integer('review_version').default(1), // criteria version for recalibration
|
reviewVersion: integer('review_version').default(1), // criteria version for recalibration
|
||||||
|
recommendation: text('recommendation'), // 'flag-malicious' | null — reviewer recommendation for admin action
|
||||||
},
|
},
|
||||||
(table) => ({
|
(table) => ({
|
||||||
skillIdx: index('idx_reviews_skill').on(table.skillId),
|
skillIdx: index('idx_reviews_skill').on(table.skillId),
|
||||||
|
|||||||
@@ -1,458 +0,0 @@
|
|||||||
#!/usr/bin/env node
|
|
||||||
/**
|
|
||||||
* Phase 5: Batch Quality Score
|
|
||||||
*
|
|
||||||
* Calculates quality scores for browse-ready skills using the same logic
|
|
||||||
* as SkillAnalyzer.calculateQuality() from services/indexer/src/analyzer.ts.
|
|
||||||
*
|
|
||||||
* Updates: quality_score, quality_details, review_status → 'auto-scored'
|
|
||||||
*
|
|
||||||
* Usage:
|
|
||||||
* DATABASE_URL=postgres://... node scripts/curation/batch-score.mjs
|
|
||||||
*
|
|
||||||
* Options:
|
|
||||||
* --dry-run Show what would change without writing
|
|
||||||
* --batch-size=N Process N skills per batch (default 100)
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { createRequire } from 'module';
|
|
||||||
import { resolve, dirname } from 'path';
|
|
||||||
import { fileURLToPath, pathToFileURL } from 'url';
|
|
||||||
|
|
||||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
||||||
|
|
||||||
// ─── Find pg module ───
|
|
||||||
let pg;
|
|
||||||
const tryPaths = [
|
|
||||||
resolve(__dirname, '../..', 'package.json'),
|
|
||||||
'/tmp/package.json',
|
|
||||||
process.env.APPDATA ? resolve(process.env.APPDATA, '..', 'Local', 'Temp', 'package.json') : null,
|
|
||||||
].filter(Boolean);
|
|
||||||
for (const p of tryPaths) {
|
|
||||||
try { pg = createRequire(p)('pg'); break; } catch {}
|
|
||||||
}
|
|
||||||
if (!pg) { console.error('pg not found. Run: npm install pg'); process.exit(1); }
|
|
||||||
|
|
||||||
// ─── Find skillhub-core (ESM-only — use direct path import) ───
|
|
||||||
let parseSkillMd, parseGenericInstructionFile, validateSkill, scanSecurity;
|
|
||||||
const corePaths = [
|
|
||||||
resolve(__dirname, '../../packages/core/dist/index.js'),
|
|
||||||
resolve(__dirname, '../../node_modules/skillhub-core/dist/index.js'),
|
|
||||||
resolve(__dirname, '../../services/indexer/node_modules/skillhub-core/dist/index.js'),
|
|
||||||
];
|
|
||||||
for (const p of corePaths) {
|
|
||||||
try {
|
|
||||||
const core = await import(pathToFileURL(p).href);
|
|
||||||
parseSkillMd = core.parseSkillMd;
|
|
||||||
parseGenericInstructionFile = core.parseGenericInstructionFile;
|
|
||||||
validateSkill = core.validateSkill;
|
|
||||||
scanSecurity = core.scanSecurity;
|
|
||||||
break;
|
|
||||||
} catch {}
|
|
||||||
}
|
|
||||||
if (!parseSkillMd) { console.error('skillhub-core not found. Run: pnpm build'); process.exit(1); }
|
|
||||||
|
|
||||||
// ─── Config ───
|
|
||||||
const DATABASE_URL = process.env.DATABASE_URL || 'postgresql://postgres:postgres@localhost:5432/skillhub';
|
|
||||||
const DRY_RUN = process.argv.includes('--dry-run');
|
|
||||||
const batchArg = process.argv.find(a => a.startsWith('--batch-size='));
|
|
||||||
const BATCH_SIZE = batchArg ? parseInt(batchArg.split('=')[1]) : 100;
|
|
||||||
|
|
||||||
// ─── Helpers ───
|
|
||||||
const n = v => Number(v ?? 0).toLocaleString('en-US');
|
|
||||||
|
|
||||||
function progress(current, total) {
|
|
||||||
const pct = ((current / total) * 100).toFixed(1);
|
|
||||||
process.stdout.write(`\r Processing ${n(current)} / ${n(total)} (${pct}%)`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Database ───
|
|
||||||
let client;
|
|
||||||
|
|
||||||
async function connect() {
|
|
||||||
client = new pg.Client({
|
|
||||||
connectionString: DATABASE_URL,
|
|
||||||
ssl: false,
|
|
||||||
connectionTimeoutMillis: 15000,
|
|
||||||
query_timeout: 600000,
|
|
||||||
keepAlive: true,
|
|
||||||
});
|
|
||||||
await client.connect();
|
|
||||||
console.log('Connected to database');
|
|
||||||
}
|
|
||||||
|
|
||||||
async function query(sql, params = []) {
|
|
||||||
return client.query(sql, params);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Quality scoring (mirrors SkillAnalyzer.calculateQuality) ───
|
|
||||||
|
|
||||||
function scoreDocumentation(skill, scripts, references) {
|
|
||||||
let score = 0;
|
|
||||||
|
|
||||||
// Has description (required)
|
|
||||||
if (skill.metadata.description && skill.metadata.description.length > 20) {
|
|
||||||
score += 20;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Content length and structure
|
|
||||||
const contentLength = skill.content.length;
|
|
||||||
if (contentLength > 500) score += 15;
|
|
||||||
else if (contentLength > 200) score += 10;
|
|
||||||
else if (contentLength > 50) score += 5;
|
|
||||||
|
|
||||||
// Has headers (good structure)
|
|
||||||
const headerCount = (skill.content.match(/^#+\s/gm) || []).length;
|
|
||||||
if (headerCount >= 3) score += 15;
|
|
||||||
else if (headerCount >= 1) score += 10;
|
|
||||||
|
|
||||||
// Has code examples
|
|
||||||
if (skill.content.includes('```')) {
|
|
||||||
score += 15;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Has version
|
|
||||||
if (skill.metadata.version) {
|
|
||||||
score += 10;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Has license
|
|
||||||
if (skill.metadata.license) {
|
|
||||||
score += 5;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Has compatibility info
|
|
||||||
if (skill.metadata.compatibility?.platforms?.length) {
|
|
||||||
score += 10;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Has scripts
|
|
||||||
if (scripts.length > 0) {
|
|
||||||
score += 5;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Has references
|
|
||||||
if (references.length > 0) {
|
|
||||||
score += 5;
|
|
||||||
}
|
|
||||||
|
|
||||||
return Math.min(100, score);
|
|
||||||
}
|
|
||||||
|
|
||||||
function scoreMaintenance(repoMeta) {
|
|
||||||
let score = 0;
|
|
||||||
|
|
||||||
// Check last update time
|
|
||||||
const lastUpdate = new Date(repoMeta.updatedAt);
|
|
||||||
const daysSinceUpdate = (Date.now() - lastUpdate.getTime()) / (1000 * 60 * 60 * 24);
|
|
||||||
|
|
||||||
if (daysSinceUpdate < 30) score += 40;
|
|
||||||
else if (daysSinceUpdate < 90) score += 30;
|
|
||||||
else if (daysSinceUpdate < 180) score += 20;
|
|
||||||
else if (daysSinceUpdate < 365) score += 10;
|
|
||||||
|
|
||||||
// Has license
|
|
||||||
if (repoMeta.license) {
|
|
||||||
score += 20;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Has description
|
|
||||||
if (repoMeta.description) {
|
|
||||||
score += 10;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Has topics (always [] since not in DB — will miss ~10 points)
|
|
||||||
if (repoMeta.topics.length > 0) {
|
|
||||||
score += 10;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Activity level (forks indicate usage)
|
|
||||||
if (repoMeta.forks >= 10) score += 20;
|
|
||||||
else if (repoMeta.forks >= 5) score += 15;
|
|
||||||
else if (repoMeta.forks >= 1) score += 10;
|
|
||||||
|
|
||||||
return Math.min(100, score);
|
|
||||||
}
|
|
||||||
|
|
||||||
function scorePopularity(repoMeta) {
|
|
||||||
const stars = repoMeta.stars;
|
|
||||||
const forks = repoMeta.forks;
|
|
||||||
|
|
||||||
let score = 0;
|
|
||||||
|
|
||||||
if (stars >= 1000) score += 50;
|
|
||||||
else if (stars >= 100) score += 40;
|
|
||||||
else if (stars >= 50) score += 30;
|
|
||||||
else if (stars >= 10) score += 20;
|
|
||||||
else if (stars >= 5) score += 10;
|
|
||||||
else if (stars >= 1) score += 5;
|
|
||||||
|
|
||||||
if (forks >= 50) score += 30;
|
|
||||||
else if (forks >= 10) score += 20;
|
|
||||||
else if (forks >= 5) score += 15;
|
|
||||||
else if (forks >= 1) score += 10;
|
|
||||||
|
|
||||||
// Bonus for relevant topics (always [] from DB — will miss ~20 points)
|
|
||||||
const relevantTopics = ['ai', 'agent', 'skill', 'claude', 'copilot', 'codex', 'llm'];
|
|
||||||
const hasRelevantTopic = repoMeta.topics.some(t =>
|
|
||||||
relevantTopics.some(rt => t.toLowerCase().includes(rt))
|
|
||||||
);
|
|
||||||
if (hasRelevantTopic) {
|
|
||||||
score += 20;
|
|
||||||
}
|
|
||||||
|
|
||||||
return Math.min(100, score);
|
|
||||||
}
|
|
||||||
|
|
||||||
function calculateQuality(skill, repoMeta, securityScore, validation, scripts, references) {
|
|
||||||
const factors = [];
|
|
||||||
|
|
||||||
// Documentation quality (30% weight)
|
|
||||||
const docScore = scoreDocumentation(skill, scripts, references);
|
|
||||||
factors.push({ name: 'documentation', score: docScore, weight: 0.3 });
|
|
||||||
|
|
||||||
// Maintenance signals (25% weight)
|
|
||||||
const maintScore = scoreMaintenance(repoMeta);
|
|
||||||
factors.push({ name: 'maintenance', score: maintScore, weight: 0.25 });
|
|
||||||
|
|
||||||
// Popularity (20% weight)
|
|
||||||
const popScore = scorePopularity(repoMeta);
|
|
||||||
factors.push({ name: 'popularity', score: popScore, weight: 0.2 });
|
|
||||||
|
|
||||||
// Security (15% weight)
|
|
||||||
factors.push({ name: 'security', score: securityScore, weight: 0.15 });
|
|
||||||
|
|
||||||
// Validation (10% weight)
|
|
||||||
const valScore = validation.isValid ? 100 : Math.max(0, 100 - validation.errors.length * 20);
|
|
||||||
factors.push({ name: 'validation', score: valScore, weight: 0.1 });
|
|
||||||
|
|
||||||
// Calculate weighted overall score
|
|
||||||
const overall = Math.round(
|
|
||||||
factors.reduce((sum, f) => sum + f.score * f.weight, 0)
|
|
||||||
);
|
|
||||||
|
|
||||||
return {
|
|
||||||
overall,
|
|
||||||
documentation: docScore,
|
|
||||||
maintenance: maintScore,
|
|
||||||
popularity: popScore,
|
|
||||||
factors,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Extract scripts/references from cached_files ───
|
|
||||||
function extractScripts(cachedFiles) {
|
|
||||||
if (!cachedFiles || !cachedFiles.items) return [];
|
|
||||||
return cachedFiles.items
|
|
||||||
.filter(item =>
|
|
||||||
!item.isBinary &&
|
|
||||||
item.name !== 'SKILL.md' &&
|
|
||||||
(item.name.endsWith('.sh') ||
|
|
||||||
item.name.endsWith('.py') ||
|
|
||||||
item.name.endsWith('.js') ||
|
|
||||||
item.name.endsWith('.ts') ||
|
|
||||||
item.name.endsWith('.ps1') ||
|
|
||||||
item.name.endsWith('.bat') ||
|
|
||||||
item.name.endsWith('.rb'))
|
|
||||||
)
|
|
||||||
.map(item => ({ name: item.name, content: item.content }));
|
|
||||||
}
|
|
||||||
|
|
||||||
function extractReferences(cachedFiles) {
|
|
||||||
if (!cachedFiles || !cachedFiles.items) return [];
|
|
||||||
return cachedFiles.items
|
|
||||||
.filter(item =>
|
|
||||||
!item.isBinary &&
|
|
||||||
item.name !== 'SKILL.md' &&
|
|
||||||
(item.name.endsWith('.md') || item.name.endsWith('.txt'))
|
|
||||||
)
|
|
||||||
.map(item => ({ name: item.name, content: item.content }));
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Main ───
|
|
||||||
async function main() {
|
|
||||||
const t0 = Date.now();
|
|
||||||
await connect();
|
|
||||||
|
|
||||||
console.log(`\n${'='.repeat(70)}`);
|
|
||||||
console.log(' BATCH QUALITY SCORE (Phase 5)');
|
|
||||||
console.log(`${'='.repeat(70)}`);
|
|
||||||
if (DRY_RUN) console.log('\n *** DRY RUN MODE — no changes will be made ***\n');
|
|
||||||
|
|
||||||
// Count skills to score
|
|
||||||
const countResult = await query(`
|
|
||||||
SELECT COUNT(*)::int AS total
|
|
||||||
FROM skills
|
|
||||||
WHERE is_blocked = false
|
|
||||||
AND is_duplicate = false
|
|
||||||
AND quality_score IS NULL
|
|
||||||
AND raw_content IS NOT NULL
|
|
||||||
`);
|
|
||||||
const total = countResult.rows[0].total;
|
|
||||||
console.log(` Skills to score: ${n(total)}`);
|
|
||||||
console.log(` Batch size: ${BATCH_SIZE}`);
|
|
||||||
|
|
||||||
if (total === 0) {
|
|
||||||
console.log(' Nothing to do — all browse-ready skills already scored');
|
|
||||||
await client.end().catch(() => {});
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (DRY_RUN) {
|
|
||||||
const sample = await query(`
|
|
||||||
SELECT id, LEFT(name, 50) AS name, github_stars
|
|
||||||
FROM skills
|
|
||||||
WHERE is_blocked = false
|
|
||||||
AND is_duplicate = false
|
|
||||||
AND is_duplicate = false
|
|
||||||
AND quality_score IS NULL
|
|
||||||
AND raw_content IS NOT NULL
|
|
||||||
ORDER BY github_stars DESC NULLS LAST
|
|
||||||
LIMIT 5
|
|
||||||
`);
|
|
||||||
console.log('\n Sample skills that would be scored:');
|
|
||||||
for (const r of sample.rows) {
|
|
||||||
console.log(` [${n(r.github_stars)}★] ${r.id} — ${r.name}`);
|
|
||||||
}
|
|
||||||
console.log(`\n [DRY RUN] Would score ${n(total)} skills`);
|
|
||||||
await client.end().catch(() => {});
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Process in batches
|
|
||||||
let processed = 0;
|
|
||||||
let errorCount = 0;
|
|
||||||
let totalScore = 0;
|
|
||||||
const scoreBuckets = { excellent: 0, good: 0, fair: 0, poor: 0 };
|
|
||||||
|
|
||||||
while (processed < total) {
|
|
||||||
const batch = await query(`
|
|
||||||
SELECT id, raw_content, cached_files, source_format,
|
|
||||||
github_stars, github_forks, license, description,
|
|
||||||
updated_at, version, compatibility, security_score
|
|
||||||
FROM skills
|
|
||||||
WHERE is_blocked = false
|
|
||||||
AND is_duplicate = false
|
|
||||||
AND is_duplicate = false
|
|
||||||
AND quality_score IS NULL
|
|
||||||
AND raw_content IS NOT NULL
|
|
||||||
ORDER BY github_stars DESC NULLS LAST
|
|
||||||
LIMIT $1
|
|
||||||
`, [BATCH_SIZE]);
|
|
||||||
|
|
||||||
if (batch.rows.length === 0) break;
|
|
||||||
|
|
||||||
for (const row of batch.rows) {
|
|
||||||
try {
|
|
||||||
// Parse the skill content
|
|
||||||
const sourceFormat = row.source_format || 'skill.md';
|
|
||||||
let skill;
|
|
||||||
if (sourceFormat === 'skill.md') {
|
|
||||||
skill = parseSkillMd(row.raw_content);
|
|
||||||
} else {
|
|
||||||
skill = parseGenericInstructionFile(row.raw_content, sourceFormat, {
|
|
||||||
name: row.description?.split(/\s+/).slice(0, 3).join('-') || 'skill',
|
|
||||||
description: row.description,
|
|
||||||
owner: '',
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate
|
|
||||||
const validation = sourceFormat === 'skill.md'
|
|
||||||
? validateSkill(skill)
|
|
||||||
: { isValid: true, errors: [], warnings: [] };
|
|
||||||
|
|
||||||
// Construct pseudo-repoMeta from DB fields
|
|
||||||
const repoMeta = {
|
|
||||||
stars: row.github_stars || 0,
|
|
||||||
forks: row.github_forks || 0,
|
|
||||||
license: row.license || null,
|
|
||||||
description: row.description || null,
|
|
||||||
updatedAt: row.updated_at ? row.updated_at.toISOString() : new Date().toISOString(),
|
|
||||||
defaultBranch: 'main',
|
|
||||||
topics: [], // NOT in DB — popularity factor will miss ~20 points
|
|
||||||
};
|
|
||||||
|
|
||||||
// Use already-computed security score (from Phase 4), default to 100 if not scanned
|
|
||||||
const securityScore = row.security_score ?? 100;
|
|
||||||
|
|
||||||
// Extract scripts/references from cached_files
|
|
||||||
const scripts = extractScripts(row.cached_files);
|
|
||||||
const references = extractReferences(row.cached_files);
|
|
||||||
|
|
||||||
// Calculate quality
|
|
||||||
const quality = calculateQuality(skill, repoMeta, securityScore, validation, scripts, references);
|
|
||||||
|
|
||||||
// Build quality_details for JSONB storage
|
|
||||||
const qualityDetails = {
|
|
||||||
documentation: quality.documentation,
|
|
||||||
maintenance: quality.maintenance,
|
|
||||||
popularity: quality.popularity,
|
|
||||||
factors: quality.factors,
|
|
||||||
};
|
|
||||||
|
|
||||||
await query(`
|
|
||||||
UPDATE skills
|
|
||||||
SET quality_score = $1,
|
|
||||||
quality_details = $2,
|
|
||||||
review_status = CASE
|
|
||||||
WHEN review_status IS NULL OR review_status = 'unreviewed'
|
|
||||||
THEN 'auto-scored'
|
|
||||||
ELSE review_status
|
|
||||||
END
|
|
||||||
WHERE id = $3
|
|
||||||
`, [quality.overall, JSON.stringify(qualityDetails), row.id]);
|
|
||||||
|
|
||||||
totalScore += quality.overall;
|
|
||||||
if (quality.overall >= 70) scoreBuckets.excellent++;
|
|
||||||
else if (quality.overall >= 50) scoreBuckets.good++;
|
|
||||||
else if (quality.overall >= 30) scoreBuckets.fair++;
|
|
||||||
else scoreBuckets.poor++;
|
|
||||||
} catch (err) {
|
|
||||||
errorCount++;
|
|
||||||
if (errorCount <= 5) {
|
|
||||||
console.error(`\n Error scoring ${row.id}: ${err.message}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
processed++;
|
|
||||||
if (processed % 100 === 0 || processed === total) {
|
|
||||||
progress(processed, total);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log('\n'); // Clear progress line
|
|
||||||
|
|
||||||
// Summary
|
|
||||||
const scored = processed - errorCount;
|
|
||||||
const avgScore = scored > 0 ? (totalScore / scored).toFixed(1) : 0;
|
|
||||||
console.log(`
|
|
||||||
┌─────────────────────────────────────────────────────┐
|
|
||||||
│ QUALITY SCORE SUMMARY │
|
|
||||||
├─────────────────────────────────────────────────────┤
|
|
||||||
│ Total scored: ${n(scored).padStart(8)} │
|
|
||||||
│ Errors (skipped): ${n(errorCount).padStart(8)} │
|
|
||||||
├─────────────────────────────────────────────────────┤
|
|
||||||
│ Excellent (70+): ${n(scoreBuckets.excellent).padStart(8)} │
|
|
||||||
│ Good (50-69): ${n(scoreBuckets.good).padStart(8)} │
|
|
||||||
│ Fair (30-49): ${n(scoreBuckets.fair).padStart(8)} │
|
|
||||||
│ Poor (<30): ${n(scoreBuckets.poor).padStart(8)} │
|
|
||||||
├─────────────────────────────────────────────────────┤
|
|
||||||
│ Avg quality score: ${avgScore.toString().padStart(8)} │
|
|
||||||
└─────────────────────────────────────────────────────┘
|
|
||||||
`);
|
|
||||||
|
|
||||||
console.log(' Note: repoMeta.topics not in DB — popularity score misses ~20 points');
|
|
||||||
|
|
||||||
const dur = ((Date.now() - t0) / 1000).toFixed(1);
|
|
||||||
console.log(`\nCompleted in ${dur}s`);
|
|
||||||
|
|
||||||
await client.end().catch(() => {});
|
|
||||||
}
|
|
||||||
|
|
||||||
main().catch(async e => {
|
|
||||||
console.error('FAILED:', e.message || e);
|
|
||||||
await client?.end().catch(() => {});
|
|
||||||
process.exit(1);
|
|
||||||
});
|
|
||||||
@@ -1,239 +0,0 @@
|
|||||||
#!/usr/bin/env node
|
|
||||||
/**
|
|
||||||
* Phase 4: Batch Security Scan
|
|
||||||
*
|
|
||||||
* Scans browse-ready skills for security issues using scanSecurity() from skillhub-core.
|
|
||||||
* Updates: security_score, security_status, last_scanned, review_status → 'auto-scored'
|
|
||||||
*
|
|
||||||
* Usage:
|
|
||||||
* DATABASE_URL=postgres://... node scripts/curation/batch-security.mjs
|
|
||||||
*
|
|
||||||
* Options:
|
|
||||||
* --dry-run Show what would change without writing
|
|
||||||
* --batch-size=N Process N skills per batch (default 100)
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { createRequire } from 'module';
|
|
||||||
import { resolve, dirname } from 'path';
|
|
||||||
import { fileURLToPath, pathToFileURL } from 'url';
|
|
||||||
|
|
||||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
||||||
|
|
||||||
// ─── Find pg module ───
|
|
||||||
let pg;
|
|
||||||
const tryPaths = [
|
|
||||||
resolve(__dirname, '../..', 'package.json'),
|
|
||||||
'/tmp/package.json',
|
|
||||||
process.env.APPDATA ? resolve(process.env.APPDATA, '..', 'Local', 'Temp', 'package.json') : null,
|
|
||||||
].filter(Boolean);
|
|
||||||
for (const p of tryPaths) {
|
|
||||||
try { pg = createRequire(p)('pg'); break; } catch {}
|
|
||||||
}
|
|
||||||
if (!pg) { console.error('pg not found. Run: npm install pg'); process.exit(1); }
|
|
||||||
|
|
||||||
// ─── Find skillhub-core (ESM-only — use direct path import) ───
|
|
||||||
let scanSecurity;
|
|
||||||
const corePaths = [
|
|
||||||
resolve(__dirname, '../../packages/core/dist/index.js'),
|
|
||||||
resolve(__dirname, '../../node_modules/skillhub-core/dist/index.js'),
|
|
||||||
resolve(__dirname, '../../services/indexer/node_modules/skillhub-core/dist/index.js'),
|
|
||||||
];
|
|
||||||
for (const p of corePaths) {
|
|
||||||
try {
|
|
||||||
const core = await import(pathToFileURL(p).href);
|
|
||||||
scanSecurity = core.scanSecurity;
|
|
||||||
break;
|
|
||||||
} catch {}
|
|
||||||
}
|
|
||||||
if (!scanSecurity) { console.error('skillhub-core not found. Run: pnpm build'); process.exit(1); }
|
|
||||||
|
|
||||||
// ─── Config ───
|
|
||||||
const DATABASE_URL = process.env.DATABASE_URL || 'postgresql://postgres:postgres@localhost:5432/skillhub';
|
|
||||||
const DRY_RUN = process.argv.includes('--dry-run');
|
|
||||||
const batchArg = process.argv.find(a => a.startsWith('--batch-size='));
|
|
||||||
const BATCH_SIZE = batchArg ? parseInt(batchArg.split('=')[1]) : 100;
|
|
||||||
|
|
||||||
// ─── Helpers ───
|
|
||||||
const n = v => Number(v ?? 0).toLocaleString('en-US');
|
|
||||||
|
|
||||||
function progress(current, total) {
|
|
||||||
const pct = ((current / total) * 100).toFixed(1);
|
|
||||||
process.stdout.write(`\r Processing ${n(current)} / ${n(total)} (${pct}%)`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Database ───
|
|
||||||
let client;
|
|
||||||
|
|
||||||
async function connect() {
|
|
||||||
client = new pg.Client({
|
|
||||||
connectionString: DATABASE_URL,
|
|
||||||
ssl: false,
|
|
||||||
connectionTimeoutMillis: 15000,
|
|
||||||
query_timeout: 600000,
|
|
||||||
keepAlive: true,
|
|
||||||
});
|
|
||||||
await client.connect();
|
|
||||||
console.log('Connected to database');
|
|
||||||
}
|
|
||||||
|
|
||||||
async function query(sql, params = []) {
|
|
||||||
return client.query(sql, params);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Extract scripts from cached_files ───
|
|
||||||
function extractScripts(cachedFiles) {
|
|
||||||
if (!cachedFiles || !cachedFiles.items) return [];
|
|
||||||
return cachedFiles.items
|
|
||||||
.filter(item =>
|
|
||||||
!item.isBinary &&
|
|
||||||
item.name !== 'SKILL.md' &&
|
|
||||||
(item.name.endsWith('.sh') ||
|
|
||||||
item.name.endsWith('.py') ||
|
|
||||||
item.name.endsWith('.js') ||
|
|
||||||
item.name.endsWith('.ts') ||
|
|
||||||
item.name.endsWith('.ps1') ||
|
|
||||||
item.name.endsWith('.bat') ||
|
|
||||||
item.name.endsWith('.rb'))
|
|
||||||
)
|
|
||||||
.map(item => ({ name: item.name, content: item.content }));
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Main ───
|
|
||||||
async function main() {
|
|
||||||
const t0 = Date.now();
|
|
||||||
await connect();
|
|
||||||
|
|
||||||
console.log(`\n${'='.repeat(70)}`);
|
|
||||||
console.log(' BATCH SECURITY SCAN (Phase 4)');
|
|
||||||
console.log(`${'='.repeat(70)}`);
|
|
||||||
if (DRY_RUN) console.log('\n *** DRY RUN MODE — no changes will be made ***\n');
|
|
||||||
|
|
||||||
// Count skills to scan
|
|
||||||
const countResult = await query(`
|
|
||||||
SELECT COUNT(*)::int AS total
|
|
||||||
FROM skills
|
|
||||||
WHERE is_blocked = false
|
|
||||||
AND is_duplicate = false
|
|
||||||
AND security_status IS NULL
|
|
||||||
AND raw_content IS NOT NULL
|
|
||||||
`);
|
|
||||||
const total = countResult.rows[0].total;
|
|
||||||
console.log(` Skills to scan: ${n(total)}`);
|
|
||||||
console.log(` Batch size: ${BATCH_SIZE}`);
|
|
||||||
|
|
||||||
if (total === 0) {
|
|
||||||
console.log(' Nothing to do — all browse-ready skills already scanned');
|
|
||||||
await client.end().catch(() => {});
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (DRY_RUN) {
|
|
||||||
// Show sample
|
|
||||||
const sample = await query(`
|
|
||||||
SELECT id, LEFT(name, 50) AS name
|
|
||||||
FROM skills
|
|
||||||
WHERE is_blocked = false
|
|
||||||
AND is_duplicate = false
|
|
||||||
AND is_duplicate = false
|
|
||||||
AND security_status IS NULL
|
|
||||||
AND raw_content IS NOT NULL
|
|
||||||
LIMIT 5
|
|
||||||
`);
|
|
||||||
console.log('\n Sample skills that would be scanned:');
|
|
||||||
for (const r of sample.rows) {
|
|
||||||
console.log(` ${r.id} — ${r.name}`);
|
|
||||||
}
|
|
||||||
console.log(`\n [DRY RUN] Would scan ${n(total)} skills`);
|
|
||||||
await client.end().catch(() => {});
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Process in batches
|
|
||||||
let processed = 0;
|
|
||||||
let passCount = 0;
|
|
||||||
let warnCount = 0;
|
|
||||||
let failCount = 0;
|
|
||||||
let errorCount = 0;
|
|
||||||
let totalScore = 0;
|
|
||||||
|
|
||||||
while (processed < total) {
|
|
||||||
const batch = await query(`
|
|
||||||
SELECT id, raw_content, cached_files
|
|
||||||
FROM skills
|
|
||||||
WHERE is_blocked = false
|
|
||||||
AND is_duplicate = false
|
|
||||||
AND is_duplicate = false
|
|
||||||
AND security_status IS NULL
|
|
||||||
AND raw_content IS NOT NULL
|
|
||||||
ORDER BY github_stars DESC NULLS LAST
|
|
||||||
LIMIT $1
|
|
||||||
`, [BATCH_SIZE]);
|
|
||||||
|
|
||||||
if (batch.rows.length === 0) break;
|
|
||||||
|
|
||||||
for (const row of batch.rows) {
|
|
||||||
try {
|
|
||||||
const scripts = extractScripts(row.cached_files);
|
|
||||||
const report = scanSecurity({ content: row.raw_content, scripts });
|
|
||||||
|
|
||||||
await query(`
|
|
||||||
UPDATE skills
|
|
||||||
SET security_score = $1,
|
|
||||||
security_status = $2,
|
|
||||||
last_scanned = NOW(),
|
|
||||||
review_status = CASE
|
|
||||||
WHEN review_status IS NULL OR review_status = 'unreviewed'
|
|
||||||
THEN 'auto-scored'
|
|
||||||
ELSE review_status
|
|
||||||
END
|
|
||||||
WHERE id = $3
|
|
||||||
`, [report.score, report.status, row.id]);
|
|
||||||
|
|
||||||
totalScore += report.score;
|
|
||||||
if (report.status === 'pass') passCount++;
|
|
||||||
else if (report.status === 'warning') warnCount++;
|
|
||||||
else failCount++;
|
|
||||||
} catch (err) {
|
|
||||||
errorCount++;
|
|
||||||
if (errorCount <= 5) {
|
|
||||||
console.error(`\n Error scanning ${row.id}: ${err.message}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
processed++;
|
|
||||||
if (processed % 100 === 0 || processed === total) {
|
|
||||||
progress(processed, total);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log('\n'); // Clear progress line
|
|
||||||
|
|
||||||
// Summary
|
|
||||||
const avgScore = processed > 0 ? (totalScore / (processed - errorCount)).toFixed(1) : 0;
|
|
||||||
console.log(`
|
|
||||||
┌─────────────────────────────────────────────────────┐
|
|
||||||
│ SECURITY SCAN SUMMARY │
|
|
||||||
├─────────────────────────────────────────────────────┤
|
|
||||||
│ Total scanned: ${n(processed).padStart(8)} │
|
|
||||||
│ Errors (skipped): ${n(errorCount).padStart(8)} │
|
|
||||||
├─────────────────────────────────────────────────────┤
|
|
||||||
│ PASS: ${n(passCount).padStart(8)} │
|
|
||||||
│ WARNING: ${n(warnCount).padStart(8)} │
|
|
||||||
│ FAIL: ${n(failCount).padStart(8)} │
|
|
||||||
├─────────────────────────────────────────────────────┤
|
|
||||||
│ Avg security score:${avgScore.toString().padStart(8)} │
|
|
||||||
└─────────────────────────────────────────────────────┘
|
|
||||||
`);
|
|
||||||
|
|
||||||
const dur = ((Date.now() - t0) / 1000).toFixed(1);
|
|
||||||
console.log(`Completed in ${dur}s`);
|
|
||||||
|
|
||||||
await client.end().catch(() => {});
|
|
||||||
}
|
|
||||||
|
|
||||||
main().catch(async e => {
|
|
||||||
console.error('FAILED:', e.message || e);
|
|
||||||
await client?.end().catch(() => {});
|
|
||||||
process.exit(1);
|
|
||||||
});
|
|
||||||
@@ -1,505 +0,0 @@
|
|||||||
#!/usr/bin/env node
|
|
||||||
/**
|
|
||||||
* Phase 2: Data Cleanup & Classification
|
|
||||||
*
|
|
||||||
* Runs all curation steps in order:
|
|
||||||
* Step 1: Compute repo_skill_count for every skill
|
|
||||||
* Step 2: Mark aggregators (repos with 50+ skills)
|
|
||||||
* Step 3: Classify remaining repos (collection / standalone / project-bound)
|
|
||||||
* Step 4: Fill missing content_hash values
|
|
||||||
* Step 5: Mark duplicates by content_hash (canonical = highest stars or oldest)
|
|
||||||
* Step 6: Detect fork marketplace repos
|
|
||||||
* Step 7: Recalculate category skill counts (browse-ready only)
|
|
||||||
* Step 8: Summary report
|
|
||||||
*
|
|
||||||
* Usage:
|
|
||||||
* DATABASE_URL=postgres://... node scripts/curation/curate.mjs
|
|
||||||
* # Or with local Docker:
|
|
||||||
* DATABASE_URL=postgresql://postgres:postgres@localhost:5432/skillhub node scripts/curation/curate.mjs
|
|
||||||
*
|
|
||||||
* Options:
|
|
||||||
* --dry-run Show what would change without writing
|
|
||||||
* --step=N Run only step N (1-8)
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { createRequire } from 'module';
|
|
||||||
import { resolve, dirname } from 'path';
|
|
||||||
import { fileURLToPath } from 'url';
|
|
||||||
|
|
||||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
||||||
|
|
||||||
// ─── Find pg module ───
|
|
||||||
let pg;
|
|
||||||
const tryPaths = [
|
|
||||||
resolve(__dirname, '../..', 'package.json'),
|
|
||||||
'/tmp/package.json',
|
|
||||||
process.env.APPDATA ? resolve(process.env.APPDATA, '..', 'Local', 'Temp', 'package.json') : null,
|
|
||||||
].filter(Boolean);
|
|
||||||
for (const p of tryPaths) {
|
|
||||||
try { pg = createRequire(p)('pg'); break; } catch {}
|
|
||||||
}
|
|
||||||
if (!pg) { console.error('pg not found. Run: npm install pg'); process.exit(1); }
|
|
||||||
|
|
||||||
// ─── Config ───
|
|
||||||
const DATABASE_URL = process.env.DATABASE_URL || 'postgresql://postgres:postgres@localhost:5432/skillhub';
|
|
||||||
const DRY_RUN = process.argv.includes('--dry-run');
|
|
||||||
const stepArg = process.argv.find(a => a.startsWith('--step='));
|
|
||||||
const ONLY_STEP = stepArg ? parseInt(stepArg.split('=')[1]) : null;
|
|
||||||
|
|
||||||
// ─── Helpers ───
|
|
||||||
function header(step, title) {
|
|
||||||
console.log(`\n${'='.repeat(70)}\n STEP ${step}: ${title}\n${'='.repeat(70)}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const n = v => Number(v ?? 0).toLocaleString('en-US');
|
|
||||||
|
|
||||||
// ─── Database ───
|
|
||||||
let client;
|
|
||||||
|
|
||||||
async function connect() {
|
|
||||||
client = new pg.Client({
|
|
||||||
connectionString: DATABASE_URL,
|
|
||||||
ssl: false,
|
|
||||||
connectionTimeoutMillis: 15000,
|
|
||||||
query_timeout: 600000, // 10 min for big updates
|
|
||||||
keepAlive: true,
|
|
||||||
});
|
|
||||||
await client.connect();
|
|
||||||
console.log('Connected to database');
|
|
||||||
}
|
|
||||||
|
|
||||||
async function query(sql, params = []) {
|
|
||||||
const result = await client.query(sql, params);
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Step 1: Compute repo_skill_count ───
|
|
||||||
async function step1_repoSkillCount() {
|
|
||||||
header(1, 'COMPUTE repo_skill_count');
|
|
||||||
|
|
||||||
const countResult = await query(`
|
|
||||||
SELECT COUNT(*)::int AS total
|
|
||||||
FROM skills
|
|
||||||
WHERE is_blocked = false AND repo_skill_count IS NULL
|
|
||||||
`);
|
|
||||||
console.log(` Skills without repo_skill_count: ${n(countResult.rows[0].total)}`);
|
|
||||||
|
|
||||||
if (DRY_RUN) { console.log(' [DRY RUN] Would update all skills'); return; }
|
|
||||||
|
|
||||||
const result = await query(`
|
|
||||||
UPDATE skills s SET repo_skill_count = sub.cnt
|
|
||||||
FROM (
|
|
||||||
SELECT github_owner, github_repo, COUNT(*)::int AS cnt
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY github_owner, github_repo
|
|
||||||
) sub
|
|
||||||
WHERE s.github_owner = sub.github_owner
|
|
||||||
AND s.github_repo = sub.github_repo
|
|
||||||
AND s.is_blocked = false
|
|
||||||
`);
|
|
||||||
console.log(` Updated: ${n(result.rowCount)} skills`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Step 2: Mark aggregators ───
|
|
||||||
async function step2_markAggregators() {
|
|
||||||
header(2, 'MARK AGGREGATORS (repos with 50+ skills)');
|
|
||||||
|
|
||||||
// Preview
|
|
||||||
const preview = await query(`
|
|
||||||
SELECT github_owner || '/' || github_repo AS repo, COUNT(*)::int AS skills,
|
|
||||||
MAX(github_stars)::int AS stars
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY github_owner, github_repo
|
|
||||||
HAVING COUNT(*) >= 50
|
|
||||||
ORDER BY skills DESC LIMIT 20
|
|
||||||
`);
|
|
||||||
console.log(` Top aggregator repos (${preview.rowCount} total):`);
|
|
||||||
for (const r of preview.rows.slice(0, 10)) {
|
|
||||||
console.log(` ${r.repo}: ${n(r.skills)} skills (${n(r.stars)} stars)`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (DRY_RUN) { console.log(' [DRY RUN] Would mark skills in these repos'); return; }
|
|
||||||
|
|
||||||
const result = await query(`
|
|
||||||
UPDATE skills s SET skill_type = 'aggregator'
|
|
||||||
FROM (
|
|
||||||
SELECT github_owner, github_repo
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY github_owner, github_repo
|
|
||||||
HAVING COUNT(*) >= 50
|
|
||||||
) agg
|
|
||||||
WHERE s.github_owner = agg.github_owner
|
|
||||||
AND s.github_repo = agg.github_repo
|
|
||||||
AND s.is_blocked = false
|
|
||||||
AND s.skill_type IS NULL
|
|
||||||
`);
|
|
||||||
console.log(` Marked as aggregator: ${n(result.rowCount)} skills`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Step 3: Classify remaining repos ───
|
|
||||||
async function step3_classifyRemaining() {
|
|
||||||
header(3, 'CLASSIFY REMAINING REPOS');
|
|
||||||
|
|
||||||
// 3a: repos with 10-49 skills, name contains marketplace/awesome/collection → aggregator
|
|
||||||
if (!DRY_RUN) {
|
|
||||||
const aggByName = await query(`
|
|
||||||
UPDATE skills s SET skill_type = 'aggregator'
|
|
||||||
FROM (
|
|
||||||
SELECT github_owner, github_repo
|
|
||||||
FROM skills WHERE is_blocked = false AND skill_type IS NULL
|
|
||||||
GROUP BY github_owner, github_repo
|
|
||||||
HAVING COUNT(*) >= 10
|
|
||||||
AND (github_repo ILIKE '%marketplace%'
|
|
||||||
OR github_repo ILIKE '%awesome%'
|
|
||||||
OR github_repo ILIKE '%collection%'
|
|
||||||
OR github_repo ILIKE '%registry%')
|
|
||||||
) agg
|
|
||||||
WHERE s.github_owner = agg.github_owner
|
|
||||||
AND s.github_repo = agg.github_repo
|
|
||||||
AND s.is_blocked = false
|
|
||||||
AND s.skill_type IS NULL
|
|
||||||
`);
|
|
||||||
console.log(` Aggregator by name (10+ & marketplace/awesome/registry): ${n(aggByName.rowCount)}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3b: repos with 3-49 skills → collection
|
|
||||||
if (!DRY_RUN) {
|
|
||||||
const collections = await query(`
|
|
||||||
UPDATE skills s SET skill_type = 'collection'
|
|
||||||
FROM (
|
|
||||||
SELECT github_owner, github_repo
|
|
||||||
FROM skills WHERE is_blocked = false AND skill_type IS NULL
|
|
||||||
GROUP BY github_owner, github_repo
|
|
||||||
HAVING COUNT(*) >= 3
|
|
||||||
) col
|
|
||||||
WHERE s.github_owner = col.github_owner
|
|
||||||
AND s.github_repo = col.github_repo
|
|
||||||
AND s.is_blocked = false
|
|
||||||
AND s.skill_type IS NULL
|
|
||||||
`);
|
|
||||||
console.log(` Collection (3+ skills in repo): ${n(collections.rowCount)}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3c: single/two-skill repos with project-bound name patterns → project-bound
|
|
||||||
if (!DRY_RUN) {
|
|
||||||
const projectBound = await query(`
|
|
||||||
UPDATE skills SET skill_type = 'project-bound'
|
|
||||||
WHERE is_blocked = false AND skill_type IS NULL
|
|
||||||
AND repo_skill_count <= 2
|
|
||||||
AND (name ILIKE '%my-%' OR name ILIKE '%my\\_%' ESCAPE '\\'
|
|
||||||
OR name ILIKE '%project%' OR name ILIKE '%team%' OR name ILIKE '%internal%'
|
|
||||||
OR name ILIKE '%.mdc' OR name ILIKE '%cursorrule%'
|
|
||||||
OR name ILIKE '%config%' OR name ILIKE '%setup%')
|
|
||||||
`);
|
|
||||||
console.log(` Project-bound (name pattern): ${n(projectBound.rowCount)}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3d: remaining → standalone
|
|
||||||
if (!DRY_RUN) {
|
|
||||||
const standalone = await query(`
|
|
||||||
UPDATE skills SET skill_type = 'standalone'
|
|
||||||
WHERE is_blocked = false AND skill_type IS NULL
|
|
||||||
`);
|
|
||||||
console.log(` Standalone (remaining): ${n(standalone.rowCount)}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Summary
|
|
||||||
const summary = await query(`
|
|
||||||
SELECT skill_type, COUNT(*)::int AS count
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY skill_type ORDER BY count DESC
|
|
||||||
`);
|
|
||||||
console.log('\n Classification summary:');
|
|
||||||
for (const r of summary.rows) {
|
|
||||||
console.log(` ${(r.skill_type || 'null').padEnd(15)} ${n(r.count)}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Step 4: Fill missing content_hash ───
|
|
||||||
async function step4_fillContentHash() {
|
|
||||||
header(4, 'FILL MISSING content_hash');
|
|
||||||
|
|
||||||
const countResult = await query(`
|
|
||||||
SELECT COUNT(*)::int AS total
|
|
||||||
FROM skills WHERE is_blocked = false AND content_hash IS NULL AND raw_content IS NOT NULL
|
|
||||||
`);
|
|
||||||
const missing = countResult.rows[0].total;
|
|
||||||
console.log(` Skills missing content_hash: ${n(missing)}`);
|
|
||||||
|
|
||||||
if (missing === 0) { console.log(' Nothing to do'); return; }
|
|
||||||
if (DRY_RUN) { console.log(' [DRY RUN] Would compute hash for these skills'); return; }
|
|
||||||
|
|
||||||
// Use md5 as a reliable hash function available in PostgreSQL
|
|
||||||
const result = await query(`
|
|
||||||
UPDATE skills SET content_hash = md5(raw_content)
|
|
||||||
WHERE is_blocked = false AND content_hash IS NULL AND raw_content IS NOT NULL
|
|
||||||
`);
|
|
||||||
console.log(` Computed content_hash (md5) for: ${n(result.rowCount)} skills`);
|
|
||||||
|
|
||||||
// Note: existing hashes use a JS numeric hash (analyzer.ts hashContent).
|
|
||||||
// We now use md5 for missing ones. For dedup purposes, same content → same hash
|
|
||||||
// regardless of algorithm, since we compare within hash groups.
|
|
||||||
// But mixed algorithms mean same content could have different hashes.
|
|
||||||
// Solution: re-hash ALL with md5 for consistency.
|
|
||||||
console.log(' Re-hashing ALL skills with md5 for consistency...');
|
|
||||||
const rehash = await query(`
|
|
||||||
UPDATE skills SET content_hash = md5(raw_content)
|
|
||||||
WHERE is_blocked = false AND raw_content IS NOT NULL
|
|
||||||
`);
|
|
||||||
console.log(` Re-hashed: ${n(rehash.rowCount)} skills`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Step 5: Mark duplicates ───
|
|
||||||
async function step5_markDuplicates() {
|
|
||||||
header(5, 'MARK DUPLICATES BY content_hash');
|
|
||||||
|
|
||||||
// Find duplicate groups
|
|
||||||
const dupGroups = await query(`
|
|
||||||
SELECT content_hash, COUNT(*)::int AS copies
|
|
||||||
FROM skills
|
|
||||||
WHERE is_blocked = false AND content_hash IS NOT NULL AND is_duplicate = false
|
|
||||||
GROUP BY content_hash
|
|
||||||
HAVING COUNT(*) > 1
|
|
||||||
ORDER BY copies DESC
|
|
||||||
`);
|
|
||||||
const totalDupGroups = dupGroups.rowCount;
|
|
||||||
const totalDupSkills = dupGroups.rows.reduce((sum, r) => sum + r.copies, 0);
|
|
||||||
const removable = dupGroups.rows.reduce((sum, r) => sum + r.copies - 1, 0);
|
|
||||||
|
|
||||||
console.log(` Duplicate groups: ${n(totalDupGroups)}`);
|
|
||||||
console.log(` Total skills in dup groups: ${n(totalDupSkills)}`);
|
|
||||||
console.log(` Removable (keeping 1 per group): ${n(removable)}`);
|
|
||||||
console.log(` Top 5 by copies:`);
|
|
||||||
for (const r of dupGroups.rows.slice(0, 5)) {
|
|
||||||
console.log(` hash=${r.content_hash}: ${r.copies} copies`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (DRY_RUN) { console.log(' [DRY RUN] Would mark duplicates'); return; }
|
|
||||||
|
|
||||||
// For each duplicate group:
|
|
||||||
// - canonical = highest github_stars, then oldest created_at
|
|
||||||
// - rest = is_duplicate = true, canonical_skill_id = canonical.id
|
|
||||||
const markResult = await query(`
|
|
||||||
WITH ranked AS (
|
|
||||||
SELECT id, content_hash,
|
|
||||||
ROW_NUMBER() OVER (
|
|
||||||
PARTITION BY content_hash
|
|
||||||
ORDER BY github_stars DESC NULLS LAST,
|
|
||||||
created_at ASC
|
|
||||||
) AS rn
|
|
||||||
FROM skills
|
|
||||||
WHERE is_blocked = false AND content_hash IS NOT NULL AND is_duplicate = false
|
|
||||||
),
|
|
||||||
canonicals AS (
|
|
||||||
SELECT content_hash, id AS canonical_id
|
|
||||||
FROM ranked WHERE rn = 1
|
|
||||||
)
|
|
||||||
UPDATE skills s
|
|
||||||
SET is_duplicate = true,
|
|
||||||
canonical_skill_id = c.canonical_id
|
|
||||||
FROM ranked r
|
|
||||||
JOIN canonicals c ON r.content_hash = c.content_hash
|
|
||||||
WHERE s.id = r.id
|
|
||||||
AND r.rn > 1
|
|
||||||
`);
|
|
||||||
console.log(` Marked as duplicate: ${n(markResult.rowCount)} skills`);
|
|
||||||
|
|
||||||
// Verify
|
|
||||||
const verify = await query(`
|
|
||||||
SELECT
|
|
||||||
COUNT(*) FILTER (WHERE is_duplicate = false)::int AS unique_skills,
|
|
||||||
COUNT(*) FILTER (WHERE is_duplicate = true)::int AS duplicates
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
`);
|
|
||||||
console.log(` After dedup: ${n(verify.rows[0].unique_skills)} unique, ${n(verify.rows[0].duplicates)} duplicates`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Step 6: Detect fork marketplace repos ───
|
|
||||||
async function step6_detectForks() {
|
|
||||||
header(6, 'DETECT FORK MARKETPLACE REPOS');
|
|
||||||
|
|
||||||
// Known fork patterns: repos with same name across many owners
|
|
||||||
const forkPatterns = await query(`
|
|
||||||
SELECT github_repo, COUNT(DISTINCT github_owner)::int AS owners,
|
|
||||||
COUNT(*)::int AS total_skills
|
|
||||||
FROM skills
|
|
||||||
WHERE is_blocked = false
|
|
||||||
AND repo_skill_count >= 20
|
|
||||||
GROUP BY github_repo
|
|
||||||
HAVING COUNT(DISTINCT github_owner) >= 3
|
|
||||||
ORDER BY owners DESC
|
|
||||||
`);
|
|
||||||
|
|
||||||
console.log(` Repo names appearing in 3+ owners (with 20+ skills each):`);
|
|
||||||
for (const r of forkPatterns.rows) {
|
|
||||||
console.log(` ${r.github_repo}: ${r.owners} owners, ${n(r.total_skills)} skills`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (DRY_RUN) { console.log(' [DRY RUN] Would mark fork repo skills'); return; }
|
|
||||||
|
|
||||||
// For fork repos, all copies should be checked as duplicates
|
|
||||||
// The canonical is already handled by step 5 (content_hash dedup)
|
|
||||||
// Here we just ensure they're typed correctly as aggregator
|
|
||||||
if (forkPatterns.rows.length > 0) {
|
|
||||||
const repoNames = forkPatterns.rows.map(r => r.github_repo);
|
|
||||||
const placeholders = repoNames.map((_, i) => `$${i + 1}`).join(',');
|
|
||||||
const markForks = await query(`
|
|
||||||
UPDATE skills SET skill_type = 'aggregator'
|
|
||||||
WHERE is_blocked = false
|
|
||||||
AND github_repo IN (${placeholders})
|
|
||||||
AND repo_skill_count >= 20
|
|
||||||
AND (skill_type IS NULL OR skill_type != 'aggregator')
|
|
||||||
`, repoNames);
|
|
||||||
console.log(` Re-classified as aggregator (fork repos): ${n(markForks.rowCount)}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Step 7: Recalculate category counts ───
|
|
||||||
async function step7_categoryCounts() {
|
|
||||||
header(7, 'RECALCULATE CATEGORY COUNTS');
|
|
||||||
|
|
||||||
// Update skill_count on each category to only count browse-ready skills
|
|
||||||
const result = await query(`
|
|
||||||
UPDATE categories c
|
|
||||||
SET skill_count = sub.cnt
|
|
||||||
FROM (
|
|
||||||
SELECT sc.category_id, COUNT(*)::int AS cnt
|
|
||||||
FROM skill_categories sc
|
|
||||||
JOIN skills s ON sc.skill_id = s.id
|
|
||||||
WHERE s.is_blocked = false
|
|
||||||
AND s.is_duplicate = false
|
|
||||||
AND (s.skill_type IS NULL OR s.skill_type != 'aggregator')
|
|
||||||
GROUP BY sc.category_id
|
|
||||||
) sub
|
|
||||||
WHERE c.id = sub.category_id
|
|
||||||
AND c.skill_count IS DISTINCT FROM sub.cnt
|
|
||||||
`);
|
|
||||||
console.log(` Updated ${result.rowCount} category counts (browse-ready only)`);
|
|
||||||
|
|
||||||
// Also zero out categories that have no browse-ready skills
|
|
||||||
const zeroResult = await query(`
|
|
||||||
UPDATE categories c
|
|
||||||
SET skill_count = 0
|
|
||||||
WHERE c.skill_count > 0
|
|
||||||
AND NOT EXISTS (
|
|
||||||
SELECT 1 FROM skill_categories sc
|
|
||||||
JOIN skills s ON sc.skill_id = s.id
|
|
||||||
WHERE sc.category_id = c.id
|
|
||||||
AND s.is_blocked = false
|
|
||||||
AND s.is_duplicate = false
|
|
||||||
AND (s.skill_type IS NULL OR s.skill_type != 'aggregator')
|
|
||||||
)
|
|
||||||
`);
|
|
||||||
if (zeroResult.rowCount > 0) {
|
|
||||||
console.log(` Zeroed ${zeroResult.rowCount} categories with no browse-ready skills`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Show category counts
|
|
||||||
const cats = await query(`
|
|
||||||
SELECT name, skill_count FROM categories
|
|
||||||
WHERE id NOT LIKE 'parent-%'
|
|
||||||
ORDER BY skill_count DESC
|
|
||||||
LIMIT 10
|
|
||||||
`);
|
|
||||||
console.log(' Top 10 categories by browse-ready count:');
|
|
||||||
for (const r of cats.rows) {
|
|
||||||
console.log(` ${n(r.skill_count).padStart(6)} ${r.name}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Step 8: Summary ───
|
|
||||||
async function step8_summary() {
|
|
||||||
header(8, 'FINAL SUMMARY');
|
|
||||||
|
|
||||||
const summary = await query(`
|
|
||||||
SELECT
|
|
||||||
COUNT(*)::int AS total,
|
|
||||||
COUNT(*) FILTER (WHERE is_duplicate = false)::int AS unique_skills,
|
|
||||||
COUNT(*) FILTER (WHERE is_duplicate = true)::int AS duplicates,
|
|
||||||
COUNT(*) FILTER (WHERE skill_type = 'standalone' AND is_duplicate = false)::int AS standalone,
|
|
||||||
COUNT(*) FILTER (WHERE skill_type = 'collection' AND is_duplicate = false)::int AS collection,
|
|
||||||
COUNT(*) FILTER (WHERE skill_type = 'aggregator' AND is_duplicate = false)::int AS aggregator,
|
|
||||||
COUNT(*) FILTER (WHERE skill_type = 'project-bound' AND is_duplicate = false)::int AS project_bound,
|
|
||||||
COUNT(*) FILTER (WHERE skill_type IS NULL AND is_duplicate = false)::int AS unclassified
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
`);
|
|
||||||
const s = summary.rows[0];
|
|
||||||
|
|
||||||
console.log(`
|
|
||||||
┌─────────────────────────────────────────────────────┐
|
|
||||||
│ CURATION SUMMARY │
|
|
||||||
├─────────────────────────────────────────────────────┤
|
|
||||||
│ Total skills: ${n(s.total).padStart(8)} │
|
|
||||||
│ Unique skills: ${n(s.unique_skills).padStart(8)} │
|
|
||||||
│ Duplicates: ${n(s.duplicates).padStart(8)} │
|
|
||||||
├─────────────────────────────────────────────────────┤
|
|
||||||
│ Unique by type: │
|
|
||||||
│ Standalone: ${n(s.standalone).padStart(8)} │
|
|
||||||
│ Collection: ${n(s.collection).padStart(8)} │
|
|
||||||
│ Aggregator: ${n(s.aggregator).padStart(8)} │
|
|
||||||
│ Project-bound: ${n(s.project_bound).padStart(8)} │
|
|
||||||
│ Unclassified: ${n(s.unclassified).padStart(8)} │
|
|
||||||
└─────────────────────────────────────────────────────┘
|
|
||||||
`);
|
|
||||||
|
|
||||||
// Interesting standalone skills
|
|
||||||
const topStandalone = await query(`
|
|
||||||
SELECT id, name, github_stars AS stars, COALESCE(download_count,0) AS dl,
|
|
||||||
LEFT(description, 70) AS description
|
|
||||||
FROM skills
|
|
||||||
WHERE is_blocked = false AND is_duplicate = false
|
|
||||||
AND skill_type = 'standalone'
|
|
||||||
ORDER BY github_stars DESC NULLS LAST
|
|
||||||
LIMIT 20
|
|
||||||
`);
|
|
||||||
console.log(' Top 20 standalone skills by stars:');
|
|
||||||
for (const r of topStandalone.rows) {
|
|
||||||
console.log(` [${n(r.stars)}★ ${n(r.dl)}↓] ${r.id}`);
|
|
||||||
console.log(` ${r.description}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Browse-ready count (what users would see)
|
|
||||||
const browseReady = await query(`
|
|
||||||
SELECT COUNT(*)::int AS count
|
|
||||||
FROM skills
|
|
||||||
WHERE is_blocked = false
|
|
||||||
AND is_duplicate = false
|
|
||||||
AND skill_type IN ('standalone', 'collection')
|
|
||||||
`);
|
|
||||||
console.log(`\n Browse-ready skills (standalone + collection, unique): ${n(browseReady.rows[0].count)}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Main ───
|
|
||||||
async function main() {
|
|
||||||
const t0 = Date.now();
|
|
||||||
await connect();
|
|
||||||
|
|
||||||
if (DRY_RUN) console.log('\n *** DRY RUN MODE — no changes will be made ***\n');
|
|
||||||
|
|
||||||
const steps = [
|
|
||||||
step1_repoSkillCount,
|
|
||||||
step2_markAggregators,
|
|
||||||
step3_classifyRemaining,
|
|
||||||
step4_fillContentHash,
|
|
||||||
step5_markDuplicates,
|
|
||||||
step6_detectForks,
|
|
||||||
step7_categoryCounts,
|
|
||||||
step8_summary,
|
|
||||||
];
|
|
||||||
|
|
||||||
for (let i = 0; i < steps.length; i++) {
|
|
||||||
if (ONLY_STEP && ONLY_STEP !== i + 1) continue;
|
|
||||||
await steps[i]();
|
|
||||||
}
|
|
||||||
|
|
||||||
const dur = ((Date.now() - t0) / 1000).toFixed(1);
|
|
||||||
console.log(`\nCompleted in ${dur}s`);
|
|
||||||
|
|
||||||
await client.end().catch(() => {});
|
|
||||||
}
|
|
||||||
|
|
||||||
main().catch(async e => {
|
|
||||||
console.error('FAILED:', e.message || e);
|
|
||||||
await client?.end().catch(() => {});
|
|
||||||
process.exit(1);
|
|
||||||
});
|
|
||||||
@@ -1,214 +0,0 @@
|
|||||||
-- ============================================================
|
|
||||||
-- Phase 2: Data Cleanup & Classification (Pure SQL version)
|
|
||||||
-- Run inside DB container:
|
|
||||||
-- psql -U postgres -d skillhub -f /tmp/curate.sql
|
|
||||||
--
|
|
||||||
-- Safe to run multiple times (idempotent).
|
|
||||||
-- ============================================================
|
|
||||||
|
|
||||||
\echo '======================================================================'
|
|
||||||
\echo ' STEP 1: COMPUTE repo_skill_count'
|
|
||||||
\echo '======================================================================'
|
|
||||||
|
|
||||||
UPDATE skills s SET repo_skill_count = sub.cnt
|
|
||||||
FROM (
|
|
||||||
SELECT github_owner, github_repo, COUNT(*)::int AS cnt
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY github_owner, github_repo
|
|
||||||
) sub
|
|
||||||
WHERE s.github_owner = sub.github_owner
|
|
||||||
AND s.github_repo = sub.github_repo
|
|
||||||
AND s.is_blocked = false;
|
|
||||||
|
|
||||||
\echo ' Step 1 done. Checking counts:'
|
|
||||||
SELECT 'repo_skill_count set' AS step,
|
|
||||||
COUNT(*) FILTER (WHERE repo_skill_count IS NOT NULL) AS done,
|
|
||||||
COUNT(*) FILTER (WHERE repo_skill_count IS NULL) AS remaining
|
|
||||||
FROM skills WHERE is_blocked = false;
|
|
||||||
|
|
||||||
|
|
||||||
\echo ''
|
|
||||||
\echo '======================================================================'
|
|
||||||
\echo ' STEP 2: MARK AGGREGATORS (repos with 50+ skills)'
|
|
||||||
\echo '======================================================================'
|
|
||||||
|
|
||||||
UPDATE skills s SET skill_type = 'aggregator'
|
|
||||||
FROM (
|
|
||||||
SELECT github_owner, github_repo
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY github_owner, github_repo
|
|
||||||
HAVING COUNT(*) >= 50
|
|
||||||
) agg
|
|
||||||
WHERE s.github_owner = agg.github_owner
|
|
||||||
AND s.github_repo = agg.github_repo
|
|
||||||
AND s.is_blocked = false
|
|
||||||
AND s.skill_type IS NULL;
|
|
||||||
|
|
||||||
\echo ' Step 2 done.'
|
|
||||||
|
|
||||||
|
|
||||||
\echo ''
|
|
||||||
\echo '======================================================================'
|
|
||||||
\echo ' STEP 3: CLASSIFY REMAINING REPOS'
|
|
||||||
\echo '======================================================================'
|
|
||||||
|
|
||||||
-- 3a: 10-49 skills + name contains marketplace/awesome/collection/registry → aggregator
|
|
||||||
UPDATE skills s SET skill_type = 'aggregator'
|
|
||||||
FROM (
|
|
||||||
SELECT github_owner, github_repo
|
|
||||||
FROM skills WHERE is_blocked = false AND skill_type IS NULL
|
|
||||||
GROUP BY github_owner, github_repo
|
|
||||||
HAVING COUNT(*) >= 10
|
|
||||||
AND (github_repo ILIKE '%marketplace%'
|
|
||||||
OR github_repo ILIKE '%awesome%'
|
|
||||||
OR github_repo ILIKE '%collection%'
|
|
||||||
OR github_repo ILIKE '%registry%')
|
|
||||||
) agg
|
|
||||||
WHERE s.github_owner = agg.github_owner
|
|
||||||
AND s.github_repo = agg.github_repo
|
|
||||||
AND s.is_blocked = false
|
|
||||||
AND s.skill_type IS NULL;
|
|
||||||
|
|
||||||
-- 3b: 3+ skills → collection
|
|
||||||
UPDATE skills s SET skill_type = 'collection'
|
|
||||||
FROM (
|
|
||||||
SELECT github_owner, github_repo
|
|
||||||
FROM skills WHERE is_blocked = false AND skill_type IS NULL
|
|
||||||
GROUP BY github_owner, github_repo
|
|
||||||
HAVING COUNT(*) >= 3
|
|
||||||
) col
|
|
||||||
WHERE s.github_owner = col.github_owner
|
|
||||||
AND s.github_repo = col.github_repo
|
|
||||||
AND s.is_blocked = false
|
|
||||||
AND s.skill_type IS NULL;
|
|
||||||
|
|
||||||
-- 3c: project-bound name patterns
|
|
||||||
UPDATE skills SET skill_type = 'project-bound'
|
|
||||||
WHERE is_blocked = false AND skill_type IS NULL
|
|
||||||
AND repo_skill_count <= 2
|
|
||||||
AND (name ILIKE '%my-%' OR name ILIKE '%my\_%' ESCAPE '\'
|
|
||||||
OR name ILIKE '%project%' OR name ILIKE '%team%' OR name ILIKE '%internal%'
|
|
||||||
OR name ILIKE '%.mdc' OR name ILIKE '%cursorrule%'
|
|
||||||
OR name ILIKE '%config%' OR name ILIKE '%setup%');
|
|
||||||
|
|
||||||
-- 3d: remaining → standalone
|
|
||||||
UPDATE skills SET skill_type = 'standalone'
|
|
||||||
WHERE is_blocked = false AND skill_type IS NULL;
|
|
||||||
|
|
||||||
\echo ' Step 3 done. Classification:'
|
|
||||||
SELECT skill_type, COUNT(*)::int AS count
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY skill_type ORDER BY count DESC;
|
|
||||||
|
|
||||||
|
|
||||||
\echo ''
|
|
||||||
\echo '======================================================================'
|
|
||||||
\echo ' STEP 4: FILL content_hash WITH md5 (for consistency)'
|
|
||||||
\echo '======================================================================'
|
|
||||||
|
|
||||||
-- Re-hash ALL with md5 for consistent dedup
|
|
||||||
UPDATE skills SET content_hash = md5(raw_content)
|
|
||||||
WHERE is_blocked = false AND raw_content IS NOT NULL;
|
|
||||||
|
|
||||||
\echo ' Step 4 done.'
|
|
||||||
SELECT 'content_hash' AS step,
|
|
||||||
COUNT(*) FILTER (WHERE content_hash IS NOT NULL) AS has_hash,
|
|
||||||
COUNT(*) FILTER (WHERE content_hash IS NULL) AS no_hash
|
|
||||||
FROM skills WHERE is_blocked = false;
|
|
||||||
|
|
||||||
|
|
||||||
\echo ''
|
|
||||||
\echo '======================================================================'
|
|
||||||
\echo ' STEP 5: MARK DUPLICATES BY content_hash'
|
|
||||||
\echo '======================================================================'
|
|
||||||
|
|
||||||
-- First reset any previous duplicate marks (for idempotency)
|
|
||||||
UPDATE skills SET is_duplicate = false, canonical_skill_id = NULL
|
|
||||||
WHERE is_blocked = false AND is_duplicate = true;
|
|
||||||
|
|
||||||
-- Mark duplicates: keep the one with most stars (or oldest) as canonical
|
|
||||||
WITH ranked AS (
|
|
||||||
SELECT id, content_hash,
|
|
||||||
ROW_NUMBER() OVER (
|
|
||||||
PARTITION BY content_hash
|
|
||||||
ORDER BY github_stars DESC NULLS LAST,
|
|
||||||
created_at ASC
|
|
||||||
) AS rn
|
|
||||||
FROM skills
|
|
||||||
WHERE is_blocked = false AND content_hash IS NOT NULL
|
|
||||||
),
|
|
||||||
canonicals AS (
|
|
||||||
SELECT content_hash, id AS canonical_id
|
|
||||||
FROM ranked WHERE rn = 1
|
|
||||||
)
|
|
||||||
UPDATE skills s
|
|
||||||
SET is_duplicate = true,
|
|
||||||
canonical_skill_id = c.canonical_id
|
|
||||||
FROM ranked r
|
|
||||||
JOIN canonicals c ON r.content_hash = c.content_hash
|
|
||||||
WHERE s.id = r.id
|
|
||||||
AND r.rn > 1;
|
|
||||||
|
|
||||||
\echo ' Step 5 done.'
|
|
||||||
SELECT 'dedup' AS step,
|
|
||||||
COUNT(*) FILTER (WHERE is_duplicate = false) AS unique_skills,
|
|
||||||
COUNT(*) FILTER (WHERE is_duplicate = true) AS duplicates
|
|
||||||
FROM skills WHERE is_blocked = false;
|
|
||||||
|
|
||||||
|
|
||||||
\echo ''
|
|
||||||
\echo '======================================================================'
|
|
||||||
\echo ' STEP 6: DETECT FORK MARKETPLACE REPOS'
|
|
||||||
\echo '======================================================================'
|
|
||||||
|
|
||||||
-- Re-classify fork repos (same repo name in 3+ owners with 20+ skills) as aggregator
|
|
||||||
UPDATE skills SET skill_type = 'aggregator'
|
|
||||||
WHERE is_blocked = false
|
|
||||||
AND repo_skill_count >= 20
|
|
||||||
AND (skill_type IS NULL OR skill_type != 'aggregator')
|
|
||||||
AND github_repo IN (
|
|
||||||
SELECT github_repo
|
|
||||||
FROM skills
|
|
||||||
WHERE is_blocked = false AND repo_skill_count >= 20
|
|
||||||
GROUP BY github_repo
|
|
||||||
HAVING COUNT(DISTINCT github_owner) >= 3
|
|
||||||
);
|
|
||||||
|
|
||||||
\echo ' Step 6 done. Fork patterns:'
|
|
||||||
SELECT github_repo, COUNT(DISTINCT github_owner)::int AS owners,
|
|
||||||
COUNT(*)::int AS total_skills
|
|
||||||
FROM skills
|
|
||||||
WHERE is_blocked = false AND repo_skill_count >= 20
|
|
||||||
GROUP BY github_repo
|
|
||||||
HAVING COUNT(DISTINCT github_owner) >= 3
|
|
||||||
ORDER BY owners DESC
|
|
||||||
LIMIT 15;
|
|
||||||
|
|
||||||
|
|
||||||
\echo ''
|
|
||||||
\echo '======================================================================'
|
|
||||||
\echo ' STEP 7: FINAL SUMMARY'
|
|
||||||
\echo '======================================================================'
|
|
||||||
|
|
||||||
SELECT
|
|
||||||
COUNT(*)::int AS total,
|
|
||||||
COUNT(*) FILTER (WHERE is_duplicate = false)::int AS unique_skills,
|
|
||||||
COUNT(*) FILTER (WHERE is_duplicate = true)::int AS duplicates,
|
|
||||||
COUNT(*) FILTER (WHERE skill_type = 'standalone' AND is_duplicate = false)::int AS standalone,
|
|
||||||
COUNT(*) FILTER (WHERE skill_type = 'collection' AND is_duplicate = false)::int AS collection_type,
|
|
||||||
COUNT(*) FILTER (WHERE skill_type = 'aggregator' AND is_duplicate = false)::int AS aggregator,
|
|
||||||
COUNT(*) FILTER (WHERE skill_type = 'project-bound' AND is_duplicate = false)::int AS project_bound,
|
|
||||||
COUNT(*) FILTER (WHERE skill_type IN ('standalone','collection') AND is_duplicate = false)::int AS browse_ready
|
|
||||||
FROM skills WHERE is_blocked = false;
|
|
||||||
|
|
||||||
\echo ''
|
|
||||||
\echo ' Top 20 standalone skills by stars:'
|
|
||||||
SELECT id, github_stars AS stars, COALESCE(download_count,0) AS dl,
|
|
||||||
LEFT(description, 70) AS description
|
|
||||||
FROM skills
|
|
||||||
WHERE is_blocked = false AND is_duplicate = false AND skill_type = 'standalone'
|
|
||||||
ORDER BY github_stars DESC NULLS LAST
|
|
||||||
LIMIT 20;
|
|
||||||
|
|
||||||
\echo ''
|
|
||||||
\echo ' DONE!'
|
|
||||||
@@ -1,652 +0,0 @@
|
|||||||
#!/usr/bin/env node
|
|
||||||
/**
|
|
||||||
* Phase 1: Database Exploration — Single-Query Version
|
|
||||||
*
|
|
||||||
* Runs ALL analytics in ONE SQL query to handle unstable connections.
|
|
||||||
*
|
|
||||||
* Usage:
|
|
||||||
* PGSSLMODE=disable DATABASE_URL=postgres://... node scripts/curation/explore.mjs
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { createRequire } from 'module';
|
|
||||||
import { writeFileSync } from 'fs';
|
|
||||||
import { resolve, dirname } from 'path';
|
|
||||||
import { fileURLToPath } from 'url';
|
|
||||||
|
|
||||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
||||||
const projectRoot = resolve(__dirname, '../..');
|
|
||||||
|
|
||||||
let pg;
|
|
||||||
const tryPaths = [
|
|
||||||
'/tmp/package.json',
|
|
||||||
process.env.APPDATA ? resolve(process.env.APPDATA, '..', 'Local', 'Temp', 'package.json') : null,
|
|
||||||
resolve(projectRoot, 'package.json'),
|
|
||||||
].filter(Boolean);
|
|
||||||
for (const p of tryPaths) {
|
|
||||||
try { pg = createRequire(p)('pg'); break; } catch {}
|
|
||||||
}
|
|
||||||
if (!pg) { console.error('pg not found. Run: cd /tmp && npm install pg'); process.exit(1); }
|
|
||||||
|
|
||||||
const DATABASE_URL = process.env.DATABASE_URL || 'postgresql://postgres:postgres@localhost:5432/skillhub';
|
|
||||||
|
|
||||||
process.on('uncaughtException', e => {
|
|
||||||
if (e.code === 'ECONNRESET') return;
|
|
||||||
console.error('Fatal:', e); process.exit(1);
|
|
||||||
});
|
|
||||||
|
|
||||||
// ─── Mega Query ──────────────────────────────────────────
|
|
||||||
// All analytics combined into one JSON result
|
|
||||||
|
|
||||||
const MEGA_SQL = `
|
|
||||||
WITH active AS (
|
|
||||||
SELECT * FROM skills WHERE is_blocked = false
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 1. Overall
|
|
||||||
overall AS (
|
|
||||||
SELECT json_build_object(
|
|
||||||
'total', (SELECT COUNT(*)::int FROM skills),
|
|
||||||
'active', (SELECT COUNT(*)::int FROM active),
|
|
||||||
'blocked', (SELECT COUNT(*) FILTER (WHERE is_blocked)::int FROM skills),
|
|
||||||
'owners', (SELECT COUNT(DISTINCT github_owner)::int FROM active),
|
|
||||||
'repos', (SELECT COUNT(DISTINCT github_owner||'/'||github_repo)::int FROM active),
|
|
||||||
'skillmd', (SELECT COUNT(*) FILTER (WHERE source_format='skill.md')::int FROM active),
|
|
||||||
'other_fmt', (SELECT COUNT(*) FILTER (WHERE source_format!='skill.md')::int FROM active),
|
|
||||||
'downloads', (SELECT COALESCE(SUM(download_count),0)::bigint FROM active),
|
|
||||||
'views', (SELECT COALESCE(SUM(view_count),0)::bigint FROM active),
|
|
||||||
'rated', (SELECT COUNT(*) FILTER (WHERE rating_count>0)::int FROM active),
|
|
||||||
'first', (SELECT MIN(created_at)::text FROM active),
|
|
||||||
'last', (SELECT MAX(created_at)::text FROM active)
|
|
||||||
) AS data
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 1b. Source formats
|
|
||||||
source_formats AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT COALESCE(source_format,'null') AS format, COUNT(*)::int AS count,
|
|
||||||
COUNT(*) FILTER (WHERE is_blocked=false)::int AS active
|
|
||||||
FROM skills GROUP BY source_format ORDER BY count DESC
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 2. Stars distribution
|
|
||||||
stars_dist AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT CASE
|
|
||||||
WHEN github_stars IS NULL OR github_stars=0 THEN '0'
|
|
||||||
WHEN github_stars BETWEEN 1 AND 5 THEN '1-5'
|
|
||||||
WHEN github_stars BETWEEN 6 AND 10 THEN '6-10'
|
|
||||||
WHEN github_stars BETWEEN 11 AND 50 THEN '11-50'
|
|
||||||
WHEN github_stars BETWEEN 51 AND 100 THEN '51-100'
|
|
||||||
WHEN github_stars BETWEEN 101 AND 500 THEN '101-500'
|
|
||||||
WHEN github_stars BETWEEN 501 AND 1000 THEN '501-1K'
|
|
||||||
WHEN github_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
|
|
||||||
WHEN github_stars BETWEEN 5001 AND 10000 THEN '5K-10K'
|
|
||||||
WHEN github_stars BETWEEN 10001 AND 50000 THEN '10K-50K'
|
|
||||||
ELSE '50K+'
|
|
||||||
END AS stars, COUNT(*)::int AS skills,
|
|
||||||
COUNT(DISTINCT github_owner)::int AS owners,
|
|
||||||
COUNT(DISTINCT github_owner||'/'||github_repo)::int AS repos
|
|
||||||
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(github_stars,0))
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 3. Downloads
|
|
||||||
dl_dist AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT CASE
|
|
||||||
WHEN COALESCE(download_count,0)=0 THEN '0'
|
|
||||||
WHEN download_count BETWEEN 1 AND 5 THEN '1-5'
|
|
||||||
WHEN download_count BETWEEN 6 AND 50 THEN '6-50'
|
|
||||||
WHEN download_count BETWEEN 51 AND 500 THEN '51-500'
|
|
||||||
ELSE '500+'
|
|
||||||
END AS range, COUNT(*)::int AS skills
|
|
||||||
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(download_count,0))
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 3b. Views
|
|
||||||
vw_dist AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT CASE
|
|
||||||
WHEN COALESCE(view_count,0)=0 THEN '0'
|
|
||||||
WHEN view_count BETWEEN 1 AND 10 THEN '1-10'
|
|
||||||
WHEN view_count BETWEEN 11 AND 100 THEN '11-100'
|
|
||||||
WHEN view_count BETWEEN 101 AND 1000 THEN '101-1K'
|
|
||||||
ELSE '1K+'
|
|
||||||
END AS range, COUNT(*)::int AS skills
|
|
||||||
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(view_count,0))
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 3c. Top downloaded
|
|
||||||
top_dl AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT id, name, download_count AS dl, view_count AS views,
|
|
||||||
github_stars AS stars, github_owner AS owner
|
|
||||||
FROM active WHERE COALESCE(download_count,0)>0
|
|
||||||
ORDER BY download_count DESC LIMIT 20
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 3d. Top viewed
|
|
||||||
top_vw AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT id, name, view_count AS views, download_count AS dl,
|
|
||||||
github_stars AS stars, github_owner AS owner
|
|
||||||
FROM active WHERE COALESCE(view_count,0)>0
|
|
||||||
ORDER BY view_count DESC LIMIT 20
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 4. Security
|
|
||||||
security AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT COALESCE(security_status,'null') AS status, COUNT(*)::int AS count
|
|
||||||
FROM active GROUP BY security_status ORDER BY count DESC
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 5. Content stats
|
|
||||||
content_stats AS (
|
|
||||||
SELECT json_build_object(
|
|
||||||
'has_content', COUNT(*) FILTER (WHERE raw_content IS NOT NULL)::int,
|
|
||||||
'no_content', COUNT(*) FILTER (WHERE raw_content IS NULL)::int,
|
|
||||||
'avg_len', ROUND(AVG(LENGTH(raw_content)) FILTER (WHERE raw_content IS NOT NULL))::int,
|
|
||||||
'max_len', MAX(LENGTH(raw_content))::int,
|
|
||||||
'median_len', PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY LENGTH(raw_content))
|
|
||||||
FILTER (WHERE raw_content IS NOT NULL)::int,
|
|
||||||
'avg_desc', ROUND(AVG(LENGTH(description)))::int,
|
|
||||||
'good_desc', COUNT(*) FILTER (WHERE LENGTH(description)>100)::int,
|
|
||||||
'short_desc', COUNT(*) FILTER (WHERE LENGTH(description)<=20)::int,
|
|
||||||
'has_ver', COUNT(*) FILTER (WHERE version IS NOT NULL)::int,
|
|
||||||
'has_lic', COUNT(*) FILTER (WHERE license IS NOT NULL)::int,
|
|
||||||
'has_auth', COUNT(*) FILTER (WHERE author IS NOT NULL)::int,
|
|
||||||
'has_hp', COUNT(*) FILTER (WHERE homepage IS NOT NULL)::int
|
|
||||||
) AS data FROM active
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 5b. Content length dist
|
|
||||||
content_len AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT CASE
|
|
||||||
WHEN raw_content IS NULL THEN 'null'
|
|
||||||
WHEN LENGTH(raw_content)=0 THEN 'empty'
|
|
||||||
WHEN LENGTH(raw_content)<200 THEN '<200'
|
|
||||||
WHEN LENGTH(raw_content)<500 THEN '200-500'
|
|
||||||
WHEN LENGTH(raw_content)<1000 THEN '500-1K'
|
|
||||||
WHEN LENGTH(raw_content)<3000 THEN '1K-3K'
|
|
||||||
WHEN LENGTH(raw_content)<5000 THEN '3K-5K'
|
|
||||||
WHEN LENGTH(raw_content)<10000 THEN '5K-10K'
|
|
||||||
WHEN LENGTH(raw_content)<50000 THEN '10K-50K'
|
|
||||||
ELSE '50K+'
|
|
||||||
END AS range, COUNT(*)::int AS skills
|
|
||||||
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(LENGTH(raw_content),-1))
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 6. Triggers
|
|
||||||
trigger_stats AS (
|
|
||||||
SELECT json_build_object(
|
|
||||||
'has_triggers', COUNT(*) FILTER (WHERE triggers IS NOT NULL)::int,
|
|
||||||
'no_triggers', COUNT(*) FILTER (WHERE triggers IS NULL)::int,
|
|
||||||
'file_pats', COUNT(*) FILTER (WHERE triggers->>'filePatterns' IS NOT NULL
|
|
||||||
AND triggers->>'filePatterns'!='[]' AND triggers->>'filePatterns'!='null')::int,
|
|
||||||
'keywords', COUNT(*) FILTER (WHERE triggers->>'keywords' IS NOT NULL
|
|
||||||
AND triggers->>'keywords'!='[]' AND triggers->>'keywords'!='null')::int,
|
|
||||||
'languages', COUNT(*) FILTER (WHERE triggers->>'languages' IS NOT NULL
|
|
||||||
AND triggers->>'languages'!='[]' AND triggers->>'languages'!='null')::int,
|
|
||||||
'has_compat', COUNT(*) FILTER (WHERE compatibility IS NOT NULL)::int,
|
|
||||||
'platforms', COUNT(*) FILTER (WHERE compatibility->>'platforms' IS NOT NULL
|
|
||||||
AND compatibility->>'platforms'!='[]' AND compatibility->>'platforms'!='null')::int
|
|
||||||
) AS data FROM active
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 7. Freshness
|
|
||||||
freshness AS (
|
|
||||||
SELECT json_build_object(
|
|
||||||
'created', (SELECT json_agg(row_to_json(t)) FROM (
|
|
||||||
SELECT CASE
|
|
||||||
WHEN created_at>NOW()-INTERVAL '7 days' THEN 'Last 7d'
|
|
||||||
WHEN created_at>NOW()-INTERVAL '30 days' THEN 'Last 30d'
|
|
||||||
WHEN created_at>NOW()-INTERVAL '90 days' THEN 'Last 90d'
|
|
||||||
WHEN created_at>NOW()-INTERVAL '180 days' THEN 'Last 180d'
|
|
||||||
ELSE 'Older'
|
|
||||||
END AS range, COUNT(*)::int AS skills
|
|
||||||
FROM active GROUP BY 1 ORDER BY MIN(NOW()-created_at)
|
|
||||||
) t),
|
|
||||||
'updated', (SELECT json_agg(row_to_json(t)) FROM (
|
|
||||||
SELECT CASE
|
|
||||||
WHEN updated_at>NOW()-INTERVAL '7 days' THEN 'Last 7d'
|
|
||||||
WHEN updated_at>NOW()-INTERVAL '30 days' THEN 'Last 30d'
|
|
||||||
WHEN updated_at>NOW()-INTERVAL '90 days' THEN 'Last 90d'
|
|
||||||
WHEN updated_at>NOW()-INTERVAL '180 days' THEN 'Last 180d'
|
|
||||||
ELSE 'Older'
|
|
||||||
END AS range, COUNT(*)::int AS skills
|
|
||||||
FROM active GROUP BY 1 ORDER BY MIN(NOW()-updated_at)
|
|
||||||
) t),
|
|
||||||
'last_dl', (SELECT json_agg(row_to_json(t)) FROM (
|
|
||||||
SELECT CASE
|
|
||||||
WHEN last_downloaded_at IS NULL THEN 'Never'
|
|
||||||
WHEN last_downloaded_at>NOW()-INTERVAL '30 days' THEN 'Last 30d'
|
|
||||||
WHEN last_downloaded_at>NOW()-INTERVAL '90 days' THEN 'Last 90d'
|
|
||||||
ELSE 'Older'
|
|
||||||
END AS range, COUNT(*)::int AS skills
|
|
||||||
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(last_downloaded_at,'1970-01-01'::timestamp))
|
|
||||||
) t)
|
|
||||||
) AS data
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 8. Top owners
|
|
||||||
top_owners_count AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT github_owner AS owner, COUNT(*)::int AS skills,
|
|
||||||
COUNT(DISTINCT github_repo)::int AS repos,
|
|
||||||
MAX(github_stars)::int AS max_stars,
|
|
||||||
COALESCE(SUM(download_count),0)::int AS dl
|
|
||||||
FROM active GROUP BY github_owner ORDER BY skills DESC LIMIT 30
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
top_owners_stars AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT github_owner AS owner, MAX(github_stars)::int AS max_stars,
|
|
||||||
COUNT(*)::int AS skills, COUNT(DISTINCT github_repo)::int AS repos
|
|
||||||
FROM active GROUP BY github_owner ORDER BY max_stars DESC LIMIT 20
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
owner_concentration AS (
|
|
||||||
SELECT json_build_object(
|
|
||||||
'top1', SUM(cnt) FILTER (WHERE rn<=1)::int,
|
|
||||||
'top5', SUM(cnt) FILTER (WHERE rn<=5)::int,
|
|
||||||
'top10', SUM(cnt) FILTER (WHERE rn<=10)::int,
|
|
||||||
'top20', SUM(cnt) FILTER (WHERE rn<=20)::int,
|
|
||||||
'top50', SUM(cnt) FILTER (WHERE rn<=50)::int,
|
|
||||||
'total', SUM(cnt)::int,
|
|
||||||
'owners', COUNT(*)::int
|
|
||||||
) AS data FROM (
|
|
||||||
SELECT github_owner, COUNT(*)::int AS cnt,
|
|
||||||
ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn
|
|
||||||
FROM active GROUP BY github_owner
|
|
||||||
) x
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 9. Multi-skill repos
|
|
||||||
aggregators AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT github_owner||'/'||github_repo AS repo, COUNT(*)::int AS skills,
|
|
||||||
MAX(github_stars)::int AS stars
|
|
||||||
FROM active GROUP BY github_owner, github_repo HAVING COUNT(*)>=20
|
|
||||||
ORDER BY skills DESC LIMIT 30
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
collections AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT github_owner||'/'||github_repo AS repo, COUNT(*)::int AS skills,
|
|
||||||
MAX(github_stars)::int AS stars
|
|
||||||
FROM active GROUP BY github_owner, github_repo HAVING COUNT(*) BETWEEN 3 AND 19
|
|
||||||
ORDER BY skills DESC LIMIT 30
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
skills_per_repo AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
WITH rc AS (SELECT COUNT(*)::int AS cnt FROM active GROUP BY github_owner, github_repo)
|
|
||||||
SELECT CASE
|
|
||||||
WHEN cnt=1 THEN '1' WHEN cnt=2 THEN '2'
|
|
||||||
WHEN cnt BETWEEN 3 AND 5 THEN '3-5' WHEN cnt BETWEEN 6 AND 10 THEN '6-10'
|
|
||||||
WHEN cnt BETWEEN 11 AND 20 THEN '11-20' WHEN cnt BETWEEN 21 AND 50 THEN '21-50'
|
|
||||||
WHEN cnt BETWEEN 51 AND 100 THEN '51-100' ELSE '100+'
|
|
||||||
END AS per_repo, COUNT(*)::int AS repos, SUM(cnt)::int AS total_skills
|
|
||||||
FROM rc GROUP BY 1 ORDER BY MIN(cnt)
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 10. Categories
|
|
||||||
categories_dist AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT c.name AS category, c.skill_count::int AS skills
|
|
||||||
FROM categories c WHERE c.id NOT LIKE 'parent-%' ORDER BY c.skill_count DESC
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
uncategorized AS (
|
|
||||||
SELECT COUNT(*)::int AS data FROM active s
|
|
||||||
WHERE NOT EXISTS (SELECT 1 FROM skill_categories sc WHERE sc.skill_id=s.id)
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 11. Flags
|
|
||||||
flags AS (
|
|
||||||
SELECT json_build_object(
|
|
||||||
'verified', COUNT(*) FILTER (WHERE is_verified)::int,
|
|
||||||
'featured', COUNT(*) FILTER (WHERE is_featured)::int,
|
|
||||||
'user_rated', COUNT(*) FILTER (WHERE rating_count>0)::int,
|
|
||||||
'avg_rating', ROUND(AVG(rating) FILTER (WHERE rating IS NOT NULL),2)::numeric,
|
|
||||||
'max_ratings', MAX(rating_count)::int
|
|
||||||
) AS data FROM active
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 12. Cross-table
|
|
||||||
cross_stats AS (
|
|
||||||
SELECT json_build_object(
|
|
||||||
'users', (SELECT COUNT(*)::int FROM users),
|
|
||||||
'ratings', (SELECT COUNT(*)::int FROM ratings),
|
|
||||||
'installs', (SELECT COUNT(*)::int FROM installations),
|
|
||||||
'favorites', (SELECT COUNT(*)::int FROM favorites),
|
|
||||||
'disc_repos', (SELECT COUNT(*)::int FROM discovered_repos),
|
|
||||||
'disc_with_skills', (SELECT COUNT(*) FILTER (WHERE has_skill_md)::int FROM discovered_repos),
|
|
||||||
'removals', (SELECT COUNT(*)::int FROM removal_requests),
|
|
||||||
'adds', (SELECT COUNT(*)::int FROM add_requests),
|
|
||||||
'subs', (SELECT COUNT(*) FILTER (WHERE unsubscribed_at IS NULL)::int FROM email_subscriptions)
|
|
||||||
) AS data
|
|
||||||
),
|
|
||||||
install_platform AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT platform, COUNT(*)::int AS count FROM installations GROUP BY platform ORDER BY count DESC
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 13. Usability signals
|
|
||||||
usability AS (
|
|
||||||
SELECT json_build_object(
|
|
||||||
'strong', COUNT(*) FILTER (WHERE LENGTH(description)>50 AND raw_content IS NOT NULL
|
|
||||||
AND LENGTH(raw_content)>500 AND security_status='pass')::int,
|
|
||||||
'file_triggers', COUNT(*) FILTER (WHERE triggers IS NOT NULL AND triggers!='{}'
|
|
||||||
AND triggers->>'filePatterns' IS NOT NULL AND triggers->>'filePatterns'!='[]')::int,
|
|
||||||
'downloaded', COUNT(*) FILTER (WHERE COALESCE(download_count,0)>0)::int,
|
|
||||||
'high_q', COUNT(*) FILTER (WHERE github_stars>=10 AND LENGTH(description)>50
|
|
||||||
AND raw_content IS NOT NULL AND security_status='pass')::int,
|
|
||||||
'premium', COUNT(*) FILTER (WHERE github_stars>=100 AND COALESCE(download_count,0)>0
|
|
||||||
AND security_status='pass')::int,
|
|
||||||
'skillmd_q', COUNT(*) FILTER (WHERE source_format='skill.md' AND LENGTH(description)>50
|
|
||||||
AND raw_content IS NOT NULL AND LENGTH(raw_content)>300 AND security_status='pass')::int
|
|
||||||
) AS data FROM active
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 14. Standalone vs project-bound
|
|
||||||
repo_types AS (
|
|
||||||
SELECT json_build_object(
|
|
||||||
'single', COUNT(*) FILTER (WHERE cnt=1)::int,
|
|
||||||
'two', COUNT(*) FILTER (WHERE cnt=2)::int,
|
|
||||||
'multi', COUNT(*) FILTER (WHERE cnt>=3)::int,
|
|
||||||
'total', COUNT(*)::int
|
|
||||||
) AS data FROM (
|
|
||||||
SELECT COUNT(*)::int AS cnt FROM active GROUP BY github_owner, github_repo
|
|
||||||
) x
|
|
||||||
),
|
|
||||||
name_patterns AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT CASE
|
|
||||||
WHEN name ILIKE '%rule%' OR name ILIKE '%config%' OR name ILIKE '%setup%' THEN 'rules/config/setup'
|
|
||||||
WHEN name ILIKE '%my-%' OR name ILIKE '%my_%' THEN 'my-prefix'
|
|
||||||
WHEN name ILIKE '%cursor%' OR name ILIKE '%claude%' OR name ILIKE '%copilot%' THEN 'tool-specific'
|
|
||||||
WHEN name ILIKE '%project%' OR name ILIKE '%team%' OR name ILIKE '%internal%' THEN 'project/team'
|
|
||||||
WHEN name ILIKE '%.mdc' OR name ILIKE '%cursorrule%' THEN 'cursor-rules'
|
|
||||||
ELSE 'generic'
|
|
||||||
END AS pattern, COUNT(*)::int AS skills
|
|
||||||
FROM active GROUP BY 1 ORDER BY skills DESC
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 15. Samples
|
|
||||||
top_by_stars AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT id, name, github_stars AS stars, COALESCE(download_count,0) AS dl,
|
|
||||||
security_status AS sec, source_format AS fmt, LEFT(description,80) AS description
|
|
||||||
FROM active ORDER BY github_stars DESC LIMIT 15
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
top_by_dl AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT id, name, download_count AS dl, view_count AS views,
|
|
||||||
github_stars AS stars, LEFT(description,80) AS description
|
|
||||||
FROM active WHERE COALESCE(download_count,0)>0
|
|
||||||
ORDER BY download_count DESC LIMIT 15
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 16. Discovered repos
|
|
||||||
discovered AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT discovered_via AS source, COUNT(*)::int AS total,
|
|
||||||
COUNT(*) FILTER (WHERE has_skill_md)::int AS with_skills,
|
|
||||||
COUNT(*) FILTER (WHERE last_scanned IS NOT NULL)::int AS scanned
|
|
||||||
FROM discovered_repos GROUP BY discovered_via ORDER BY total DESC
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 18. Cache
|
|
||||||
cache_stats AS (
|
|
||||||
SELECT json_build_object(
|
|
||||||
'cached', COUNT(*) FILTER (WHERE cached_files IS NOT NULL)::int,
|
|
||||||
'not_cached', COUNT(*) FILTER (WHERE cached_files IS NULL)::int
|
|
||||||
) AS data FROM active
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 19. Branches
|
|
||||||
branches AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT COALESCE(branch,'null') AS branch, COUNT(*)::int AS count
|
|
||||||
FROM active GROUP BY branch ORDER BY count DESC LIMIT 10
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
|
|
||||||
-- 20. Duplicates
|
|
||||||
exact_dupes AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT name, LEFT(description,60) AS desc, COUNT(*)::int AS copies,
|
|
||||||
STRING_AGG(DISTINCT github_owner,', ') AS owners
|
|
||||||
FROM active WHERE description IS NOT NULL AND LENGTH(description)>10
|
|
||||||
GROUP BY name, description HAVING COUNT(*)>1
|
|
||||||
ORDER BY copies DESC LIMIT 15
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
hash_dupes AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT content_hash, COUNT(*)::int AS copies,
|
|
||||||
MIN(name) AS sample, STRING_AGG(DISTINCT github_owner,', ') AS owners
|
|
||||||
FROM active WHERE content_hash IS NOT NULL
|
|
||||||
GROUP BY content_hash HAVING COUNT(*)>2
|
|
||||||
ORDER BY copies DESC LIMIT 15
|
|
||||||
) t
|
|
||||||
)
|
|
||||||
|
|
||||||
-- Final: combine everything into one JSON
|
|
||||||
SELECT json_build_object(
|
|
||||||
'overall', (SELECT data FROM overall),
|
|
||||||
'sourceFormats', (SELECT data FROM source_formats),
|
|
||||||
'starsDist', (SELECT data FROM stars_dist),
|
|
||||||
'dlDist', (SELECT data FROM dl_dist),
|
|
||||||
'vwDist', (SELECT data FROM vw_dist),
|
|
||||||
'topDL', (SELECT data FROM top_dl),
|
|
||||||
'topVW', (SELECT data FROM top_vw),
|
|
||||||
'security', (SELECT data FROM security),
|
|
||||||
'contentStats', (SELECT data FROM content_stats),
|
|
||||||
'contentLen', (SELECT data FROM content_len),
|
|
||||||
'triggerStats', (SELECT data FROM trigger_stats),
|
|
||||||
'freshness', (SELECT data FROM freshness),
|
|
||||||
'topOwnersCount', (SELECT data FROM top_owners_count),
|
|
||||||
'topOwnersStars', (SELECT data FROM top_owners_stars),
|
|
||||||
'ownerConcentration', (SELECT data FROM owner_concentration),
|
|
||||||
'aggregators', (SELECT data FROM aggregators),
|
|
||||||
'collections', (SELECT data FROM collections),
|
|
||||||
'skillsPerRepo', (SELECT data FROM skills_per_repo),
|
|
||||||
'categories', (SELECT data FROM categories_dist),
|
|
||||||
'uncategorized', (SELECT data FROM uncategorized),
|
|
||||||
'flags', (SELECT data FROM flags),
|
|
||||||
'crossStats', (SELECT data FROM cross_stats),
|
|
||||||
'installPlatform', (SELECT data FROM install_platform),
|
|
||||||
'usability', (SELECT data FROM usability),
|
|
||||||
'repoTypes', (SELECT data FROM repo_types),
|
|
||||||
'namePatterns', (SELECT data FROM name_patterns),
|
|
||||||
'topByStars', (SELECT data FROM top_by_stars),
|
|
||||||
'topByDL', (SELECT data FROM top_by_dl),
|
|
||||||
'discovered', (SELECT data FROM discovered),
|
|
||||||
'cacheStats', (SELECT data FROM cache_stats),
|
|
||||||
'branches', (SELECT data FROM branches),
|
|
||||||
'exactDupes', (SELECT data FROM exact_dupes),
|
|
||||||
'hashDupes', (SELECT data FROM hash_dupes)
|
|
||||||
) AS report;
|
|
||||||
`;
|
|
||||||
|
|
||||||
// ─── Helpers ─────────────────────────────────────────────
|
|
||||||
|
|
||||||
function header(title) {
|
|
||||||
console.log(`\n${'='.repeat(70)}\n ${title}\n${'='.repeat(70)}`);
|
|
||||||
}
|
|
||||||
function sub(title) {
|
|
||||||
console.log(`\n -- ${title} ${'─'.repeat(Math.max(0, 58 - title.length))}`);
|
|
||||||
}
|
|
||||||
function tbl(rows, max = 30) {
|
|
||||||
if (!rows || rows.length === 0) { console.log(' (no data)'); return; }
|
|
||||||
const display = rows.slice(0, max);
|
|
||||||
const keys = Object.keys(display[0]);
|
|
||||||
const w = keys.map(k => Math.max(k.length, ...display.map(r => String(r[k] ?? '').length)));
|
|
||||||
console.log(' ' + keys.map((k, i) => k.padEnd(w[i])).join(' '));
|
|
||||||
console.log(' ' + w.map(x => '-'.repeat(x)).join(' '));
|
|
||||||
for (const row of display)
|
|
||||||
console.log(' ' + keys.map((k, i) => String(row[k] ?? '').padEnd(w[i])).join(' '));
|
|
||||||
if (rows.length > max) console.log(` ... and ${rows.length - max} more`);
|
|
||||||
}
|
|
||||||
const n = v => Number(v ?? 0).toLocaleString('en-US');
|
|
||||||
const p = (a, b) => (Number(b ?? 1) === 0 ? '0.0%' : (Number(a ?? 0) / Number(b) * 100).toFixed(1) + '%');
|
|
||||||
|
|
||||||
// ─── Run ─────────────────────────────────────────────────
|
|
||||||
|
|
||||||
async function run() {
|
|
||||||
const t0 = Date.now();
|
|
||||||
console.log('Connecting and running mega-query (this may take 30-60s)...');
|
|
||||||
|
|
||||||
const c = new pg.Client({ connectionString: DATABASE_URL, ssl: false,
|
|
||||||
connectionTimeoutMillis: 15000, query_timeout: 120000,
|
|
||||||
keepAlive: true, keepAliveInitialDelayMillis: 1000 });
|
|
||||||
await c.connect();
|
|
||||||
const result = await c.query(MEGA_SQL);
|
|
||||||
await c.end().catch(() => {});
|
|
||||||
|
|
||||||
const r = result.rows[0].report;
|
|
||||||
const dur = ((Date.now() - t0) / 1000).toFixed(1);
|
|
||||||
console.log(`Query completed in ${dur}s\n`);
|
|
||||||
|
|
||||||
// ── Display ──
|
|
||||||
const total = r.overall.active;
|
|
||||||
|
|
||||||
header('1. OVERALL STATISTICS');
|
|
||||||
const ov = r.overall;
|
|
||||||
console.log(` Total skills: ${n(ov.total)}`);
|
|
||||||
console.log(` Active (not blocked): ${n(ov.active)}`);
|
|
||||||
console.log(` Blocked: ${n(ov.blocked)}`);
|
|
||||||
console.log(` Unique owners: ${n(ov.owners)}`);
|
|
||||||
console.log(` Unique repos: ${n(ov.repos)}`);
|
|
||||||
console.log(` SKILL.md format: ${n(ov.skillmd)} (${p(ov.skillmd, ov.total)})`);
|
|
||||||
console.log(` Other formats: ${n(ov.other_fmt)} (${p(ov.other_fmt, ov.total)})`);
|
|
||||||
console.log(` Total downloads: ${n(ov.downloads)}`);
|
|
||||||
console.log(` Total views: ${n(ov.views)}`);
|
|
||||||
console.log(` Rated skills: ${n(ov.rated)}`);
|
|
||||||
console.log(` Date range: ${ov.first} -> ${ov.last}`);
|
|
||||||
sub('Source Formats');
|
|
||||||
tbl(r.sourceFormats);
|
|
||||||
|
|
||||||
header('2. GITHUB STARS');
|
|
||||||
tbl(r.starsDist);
|
|
||||||
|
|
||||||
header('3. ENGAGEMENT');
|
|
||||||
sub('Downloads'); tbl(r.dlDist);
|
|
||||||
sub('Views'); tbl(r.vwDist);
|
|
||||||
sub('Top 20 Downloaded'); tbl(r.topDL);
|
|
||||||
sub('Top 20 Viewed'); tbl(r.topVW);
|
|
||||||
|
|
||||||
header('4. SECURITY');
|
|
||||||
tbl(r.security);
|
|
||||||
|
|
||||||
header('5. CONTENT');
|
|
||||||
const cs = r.contentStats;
|
|
||||||
console.log(` Has content: ${n(cs.has_content)} (${p(cs.has_content, total)})`);
|
|
||||||
console.log(` No content: ${n(cs.no_content)} (${p(cs.no_content, total)})`);
|
|
||||||
console.log(` Avg length: ${n(cs.avg_len)} | Median: ${n(cs.median_len)} | Max: ${n(cs.max_len)}`);
|
|
||||||
console.log(` Avg desc: ${n(cs.avg_desc)} | Good(>100): ${n(cs.good_desc)} | Short(<=20): ${n(cs.short_desc)}`);
|
|
||||||
console.log(` Version: ${n(cs.has_ver)} | License: ${n(cs.has_lic)} | Author: ${n(cs.has_auth)} | HP: ${n(cs.has_hp)}`);
|
|
||||||
sub('Content Length Dist');
|
|
||||||
tbl(r.contentLen);
|
|
||||||
|
|
||||||
header('6. TRIGGERS');
|
|
||||||
const tr = r.triggerStats;
|
|
||||||
console.log(` Has triggers: ${n(tr.has_triggers)} (${p(tr.has_triggers, total)})`);
|
|
||||||
console.log(` filePatterns: ${n(tr.file_pats)} | keywords: ${n(tr.keywords)} | languages: ${n(tr.languages)}`);
|
|
||||||
console.log(` Compatibility: ${n(tr.has_compat)} | platforms: ${n(tr.platforms)}`);
|
|
||||||
|
|
||||||
header('7. FRESHNESS');
|
|
||||||
sub('Created'); tbl(r.freshness.created);
|
|
||||||
sub('Updated'); tbl(r.freshness.updated);
|
|
||||||
sub('Last DL'); tbl(r.freshness.last_dl);
|
|
||||||
|
|
||||||
header('8. TOP OWNERS');
|
|
||||||
sub('By Skill Count'); tbl(r.topOwnersCount);
|
|
||||||
sub('By Stars'); tbl(r.topOwnersStars);
|
|
||||||
sub('Concentration');
|
|
||||||
const oc = r.ownerConcentration;
|
|
||||||
console.log(` Top 1: ${n(oc.top1)} (${p(oc.top1, oc.total)}) | Top 5: ${n(oc.top5)} (${p(oc.top5, oc.total)}) | Top 10: ${n(oc.top10)} (${p(oc.top10, oc.total)})`);
|
|
||||||
console.log(` Top 20: ${n(oc.top20)} (${p(oc.top20, oc.total)}) | Top 50: ${n(oc.top50)} (${p(oc.top50, oc.total)}) | Total owners: ${n(oc.owners)}`);
|
|
||||||
|
|
||||||
header('9. MULTI-SKILL REPOS');
|
|
||||||
sub('Aggregators (20+)'); tbl(r.aggregators);
|
|
||||||
sub('Collections (3-19)'); tbl(r.collections);
|
|
||||||
sub('Skills-per-Repo'); tbl(r.skillsPerRepo);
|
|
||||||
|
|
||||||
header('10. CATEGORIES');
|
|
||||||
tbl(r.categories);
|
|
||||||
console.log(`\n Uncategorized: ${n(r.uncategorized)}`);
|
|
||||||
|
|
||||||
header('11. FLAGS');
|
|
||||||
const fl = r.flags;
|
|
||||||
console.log(` Verified: ${n(fl.verified)} | Featured: ${n(fl.featured)} | Rated: ${n(fl.user_rated)} | Avg: ${fl.avg_rating ?? 'N/A'}`);
|
|
||||||
|
|
||||||
header('12. CROSS-TABLE');
|
|
||||||
const xt = r.crossStats;
|
|
||||||
console.log(` Users: ${n(xt.users)} | Ratings: ${n(xt.ratings)} | Installs: ${n(xt.installs)} | Favs: ${n(xt.favorites)}`);
|
|
||||||
console.log(` Discovered: ${n(xt.disc_repos)} (${n(xt.disc_with_skills)} with skills) | Subs: ${n(xt.subs)}`);
|
|
||||||
sub('Install Platform'); tbl(r.installPlatform);
|
|
||||||
|
|
||||||
header('13. USABILITY SIGNALS');
|
|
||||||
const us = r.usability;
|
|
||||||
console.log(` Strong standalone: ${n(us.strong)} (desc>50 + content>500 + sec=pass)`);
|
|
||||||
console.log(` File triggers: ${n(us.file_triggers)}`);
|
|
||||||
console.log(` Ever downloaded: ${n(us.downloaded)}`);
|
|
||||||
console.log(` High quality: ${n(us.high_q)} (stars>=10 + desc + sec)`);
|
|
||||||
console.log(` Premium: ${n(us.premium)} (stars>=100 + dl>0 + sec)`);
|
|
||||||
console.log(` SKILL.md quality: ${n(us.skillmd_q)}`);
|
|
||||||
|
|
||||||
header('14. STANDALONE vs PROJECT-BOUND');
|
|
||||||
const sm = r.repoTypes;
|
|
||||||
console.log(` Single: ${n(sm.single)} (${p(sm.single,sm.total)}) | Two: ${n(sm.two)} | Multi: ${n(sm.multi)} | Total repos: ${n(sm.total)}`);
|
|
||||||
sub('Name Patterns'); tbl(r.namePatterns);
|
|
||||||
|
|
||||||
header('15. SAMPLES');
|
|
||||||
sub('Top by Stars'); tbl(r.topByStars);
|
|
||||||
sub('Top by DL'); tbl(r.topByDL);
|
|
||||||
|
|
||||||
header('16. DISCOVERED REPOS');
|
|
||||||
tbl(r.discovered);
|
|
||||||
|
|
||||||
header('17. CACHE');
|
|
||||||
const cf = r.cacheStats;
|
|
||||||
console.log(` Cached: ${n(cf.cached)} (${p(cf.cached,total)}) | Not: ${n(cf.not_cached)}`);
|
|
||||||
|
|
||||||
header('19. BRANCHES');
|
|
||||||
tbl(r.branches);
|
|
||||||
|
|
||||||
header('20. DUPLICATES');
|
|
||||||
sub('Same Name+Desc'); tbl(r.exactDupes);
|
|
||||||
sub('Same Hash (3+)'); tbl(r.hashDupes);
|
|
||||||
|
|
||||||
header(`COMPLETE (${dur}s)`);
|
|
||||||
|
|
||||||
// Save JSON
|
|
||||||
r.generatedAt = new Date().toISOString();
|
|
||||||
r.durationSeconds = parseFloat(dur);
|
|
||||||
const reportPath = resolve(__dirname, 'explore-report.json');
|
|
||||||
writeFileSync(reportPath, JSON.stringify(r, null, 2), 'utf-8');
|
|
||||||
console.log(`\n Report saved: ${reportPath}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
run().catch(async e => {
|
|
||||||
console.error('FAILED:', e.message || e);
|
|
||||||
process.exit(1);
|
|
||||||
});
|
|
||||||
@@ -1,258 +0,0 @@
|
|||||||
-- Phase 1: Database Exploration - Run inside container:
|
|
||||||
-- psql -U postgres skillhub -t -A -f /tmp/explore.sql > /tmp/report.json
|
|
||||||
-- Or paste this SQL directly in psql
|
|
||||||
|
|
||||||
WITH active AS (SELECT * FROM skills WHERE is_blocked = false),
|
|
||||||
overall AS (
|
|
||||||
SELECT json_build_object(
|
|
||||||
'total', (SELECT COUNT(*)::int FROM skills),
|
|
||||||
'active', (SELECT COUNT(*)::int FROM active),
|
|
||||||
'blocked', (SELECT COUNT(*) FILTER (WHERE is_blocked)::int FROM skills),
|
|
||||||
'owners', (SELECT COUNT(DISTINCT github_owner)::int FROM active),
|
|
||||||
'repos', (SELECT COUNT(DISTINCT github_owner||'/'||github_repo)::int FROM active),
|
|
||||||
'skillmd', (SELECT COUNT(*) FILTER (WHERE source_format='skill.md')::int FROM active),
|
|
||||||
'other_fmt', (SELECT COUNT(*) FILTER (WHERE source_format!='skill.md')::int FROM active),
|
|
||||||
'downloads', (SELECT COALESCE(SUM(download_count),0)::bigint FROM active),
|
|
||||||
'views', (SELECT COALESCE(SUM(view_count),0)::bigint FROM active),
|
|
||||||
'rated', (SELECT COUNT(*) FILTER (WHERE rating_count>0)::int FROM active),
|
|
||||||
'first', (SELECT MIN(created_at)::text FROM active),
|
|
||||||
'last', (SELECT MAX(created_at)::text FROM active)
|
|
||||||
) AS data
|
|
||||||
),
|
|
||||||
source_formats AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT COALESCE(source_format,'null') AS format, COUNT(*)::int AS count
|
|
||||||
FROM skills GROUP BY source_format ORDER BY count DESC
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
stars_dist AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT CASE
|
|
||||||
WHEN github_stars IS NULL OR github_stars=0 THEN '0'
|
|
||||||
WHEN github_stars BETWEEN 1 AND 10 THEN '1-10'
|
|
||||||
WHEN github_stars BETWEEN 11 AND 50 THEN '11-50'
|
|
||||||
WHEN github_stars BETWEEN 51 AND 100 THEN '51-100'
|
|
||||||
WHEN github_stars BETWEEN 101 AND 500 THEN '101-500'
|
|
||||||
WHEN github_stars BETWEEN 501 AND 1000 THEN '501-1K'
|
|
||||||
WHEN github_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
|
|
||||||
WHEN github_stars BETWEEN 5001 AND 50000 THEN '5K-50K'
|
|
||||||
ELSE '50K+'
|
|
||||||
END AS stars, COUNT(*)::int AS skills,
|
|
||||||
COUNT(DISTINCT github_owner)::int AS owners
|
|
||||||
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(github_stars,0))
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
dl_dist AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT CASE
|
|
||||||
WHEN COALESCE(download_count,0)=0 THEN '0'
|
|
||||||
WHEN download_count BETWEEN 1 AND 10 THEN '1-10'
|
|
||||||
WHEN download_count BETWEEN 11 AND 100 THEN '11-100'
|
|
||||||
ELSE '100+'
|
|
||||||
END AS range, COUNT(*)::int AS skills
|
|
||||||
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(download_count,0))
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
top_dl AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT id, name, download_count AS dl, github_stars AS stars, github_owner AS owner
|
|
||||||
FROM active WHERE COALESCE(download_count,0)>0 ORDER BY download_count DESC LIMIT 20
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
top_vw AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT id, name, view_count AS views, github_stars AS stars, github_owner AS owner
|
|
||||||
FROM active WHERE COALESCE(view_count,0)>0 ORDER BY view_count DESC LIMIT 20
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
security AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT COALESCE(security_status,'null') AS status, COUNT(*)::int AS count
|
|
||||||
FROM active GROUP BY security_status ORDER BY count DESC
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
content_stats AS (
|
|
||||||
SELECT json_build_object(
|
|
||||||
'has_content', COUNT(*) FILTER (WHERE raw_content IS NOT NULL)::int,
|
|
||||||
'no_content', COUNT(*) FILTER (WHERE raw_content IS NULL)::int,
|
|
||||||
'avg_len', ROUND(AVG(LENGTH(raw_content)) FILTER (WHERE raw_content IS NOT NULL))::int,
|
|
||||||
'max_len', MAX(LENGTH(raw_content))::int,
|
|
||||||
'median_len', PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY LENGTH(raw_content)) FILTER (WHERE raw_content IS NOT NULL)::int,
|
|
||||||
'avg_desc', ROUND(AVG(LENGTH(description)))::int,
|
|
||||||
'good_desc', COUNT(*) FILTER (WHERE LENGTH(description)>100)::int,
|
|
||||||
'short_desc', COUNT(*) FILTER (WHERE LENGTH(description)<=20)::int,
|
|
||||||
'has_ver', COUNT(*) FILTER (WHERE version IS NOT NULL)::int,
|
|
||||||
'has_lic', COUNT(*) FILTER (WHERE license IS NOT NULL)::int,
|
|
||||||
'has_auth', COUNT(*) FILTER (WHERE author IS NOT NULL)::int
|
|
||||||
) AS data FROM active
|
|
||||||
),
|
|
||||||
content_len AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT CASE
|
|
||||||
WHEN raw_content IS NULL THEN 'null'
|
|
||||||
WHEN LENGTH(raw_content)<200 THEN '<200'
|
|
||||||
WHEN LENGTH(raw_content)<1000 THEN '200-1K'
|
|
||||||
WHEN LENGTH(raw_content)<5000 THEN '1K-5K'
|
|
||||||
WHEN LENGTH(raw_content)<10000 THEN '5K-10K'
|
|
||||||
WHEN LENGTH(raw_content)<50000 THEN '10K-50K'
|
|
||||||
ELSE '50K+'
|
|
||||||
END AS range, COUNT(*)::int AS skills
|
|
||||||
FROM active GROUP BY 1 ORDER BY MIN(COALESCE(LENGTH(raw_content),-1))
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
trigger_stats AS (
|
|
||||||
SELECT json_build_object(
|
|
||||||
'has_triggers', COUNT(*) FILTER (WHERE triggers IS NOT NULL)::int,
|
|
||||||
'no_triggers', COUNT(*) FILTER (WHERE triggers IS NULL)::int,
|
|
||||||
'file_pats', COUNT(*) FILTER (WHERE triggers->>'filePatterns' IS NOT NULL AND triggers->>'filePatterns'!='[]')::int,
|
|
||||||
'keywords', COUNT(*) FILTER (WHERE triggers->>'keywords' IS NOT NULL AND triggers->>'keywords'!='[]')::int,
|
|
||||||
'platforms', COUNT(*) FILTER (WHERE compatibility->>'platforms' IS NOT NULL AND compatibility->>'platforms'!='[]')::int
|
|
||||||
) AS data FROM active
|
|
||||||
),
|
|
||||||
freshness AS (
|
|
||||||
SELECT json_build_object(
|
|
||||||
'created', (SELECT json_agg(row_to_json(t)) FROM (
|
|
||||||
SELECT CASE WHEN created_at>NOW()-INTERVAL '7d' THEN '7d' WHEN created_at>NOW()-INTERVAL '30d' THEN '30d'
|
|
||||||
WHEN created_at>NOW()-INTERVAL '90d' THEN '90d' ELSE 'older' END AS range, COUNT(*)::int AS skills
|
|
||||||
FROM active GROUP BY 1 ORDER BY MIN(NOW()-created_at)) t),
|
|
||||||
'updated', (SELECT json_agg(row_to_json(t)) FROM (
|
|
||||||
SELECT CASE WHEN updated_at>NOW()-INTERVAL '7d' THEN '7d' WHEN updated_at>NOW()-INTERVAL '30d' THEN '30d'
|
|
||||||
WHEN updated_at>NOW()-INTERVAL '90d' THEN '90d' ELSE 'older' END AS range, COUNT(*)::int AS skills
|
|
||||||
FROM active GROUP BY 1 ORDER BY MIN(NOW()-updated_at)) t)
|
|
||||||
) AS data
|
|
||||||
),
|
|
||||||
top_owners AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT github_owner AS owner, COUNT(*)::int AS skills, COUNT(DISTINCT github_repo)::int AS repos,
|
|
||||||
MAX(github_stars)::int AS max_stars, COALESCE(SUM(download_count),0)::int AS dl
|
|
||||||
FROM active GROUP BY github_owner ORDER BY skills DESC LIMIT 30
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
owner_concentration AS (
|
|
||||||
SELECT json_build_object(
|
|
||||||
'top1', SUM(cnt) FILTER (WHERE rn<=1)::int, 'top5', SUM(cnt) FILTER (WHERE rn<=5)::int,
|
|
||||||
'top10', SUM(cnt) FILTER (WHERE rn<=10)::int, 'top20', SUM(cnt) FILTER (WHERE rn<=20)::int,
|
|
||||||
'total', SUM(cnt)::int, 'owners', COUNT(*)::int
|
|
||||||
) AS data FROM (
|
|
||||||
SELECT github_owner, COUNT(*)::int AS cnt, ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn
|
|
||||||
FROM active GROUP BY github_owner
|
|
||||||
) x
|
|
||||||
),
|
|
||||||
aggregators AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT github_owner||'/'||github_repo AS repo, COUNT(*)::int AS skills, MAX(github_stars)::int AS stars
|
|
||||||
FROM active GROUP BY github_owner, github_repo HAVING COUNT(*)>=20 ORDER BY skills DESC LIMIT 20
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
collections AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT github_owner||'/'||github_repo AS repo, COUNT(*)::int AS skills, MAX(github_stars)::int AS stars
|
|
||||||
FROM active GROUP BY github_owner, github_repo HAVING COUNT(*) BETWEEN 3 AND 19 ORDER BY skills DESC LIMIT 20
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
skills_per_repo AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
WITH rc AS (SELECT COUNT(*)::int AS cnt FROM active GROUP BY github_owner, github_repo)
|
|
||||||
SELECT CASE WHEN cnt=1 THEN '1' WHEN cnt=2 THEN '2' WHEN cnt BETWEEN 3 AND 10 THEN '3-10'
|
|
||||||
WHEN cnt BETWEEN 11 AND 50 THEN '11-50' ELSE '50+' END AS per_repo,
|
|
||||||
COUNT(*)::int AS repos, SUM(cnt)::int AS total_skills
|
|
||||||
FROM rc GROUP BY 1 ORDER BY MIN(cnt)
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
categories_dist AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT c.name AS category, c.skill_count::int AS skills
|
|
||||||
FROM categories c WHERE c.id NOT LIKE 'parent-%' ORDER BY c.skill_count DESC
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
flags AS (
|
|
||||||
SELECT json_build_object(
|
|
||||||
'verified', COUNT(*) FILTER (WHERE is_verified)::int,
|
|
||||||
'featured', COUNT(*) FILTER (WHERE is_featured)::int,
|
|
||||||
'user_rated', COUNT(*) FILTER (WHERE rating_count>0)::int
|
|
||||||
) AS data FROM active
|
|
||||||
),
|
|
||||||
cross_stats AS (
|
|
||||||
SELECT json_build_object(
|
|
||||||
'users', (SELECT COUNT(*)::int FROM users),
|
|
||||||
'ratings', (SELECT COUNT(*)::int FROM ratings),
|
|
||||||
'installs', (SELECT COUNT(*)::int FROM installations),
|
|
||||||
'favorites', (SELECT COUNT(*)::int FROM favorites),
|
|
||||||
'disc_repos', (SELECT COUNT(*)::int FROM discovered_repos),
|
|
||||||
'subs', (SELECT COUNT(*) FILTER (WHERE unsubscribed_at IS NULL)::int FROM email_subscriptions)
|
|
||||||
) AS data
|
|
||||||
),
|
|
||||||
usability AS (
|
|
||||||
SELECT json_build_object(
|
|
||||||
'strong', COUNT(*) FILTER (WHERE LENGTH(description)>50 AND raw_content IS NOT NULL AND LENGTH(raw_content)>500 AND security_status='pass')::int,
|
|
||||||
'downloaded', COUNT(*) FILTER (WHERE COALESCE(download_count,0)>0)::int,
|
|
||||||
'high_q', COUNT(*) FILTER (WHERE github_stars>=10 AND LENGTH(description)>50 AND raw_content IS NOT NULL AND security_status='pass')::int,
|
|
||||||
'premium', COUNT(*) FILTER (WHERE github_stars>=100 AND COALESCE(download_count,0)>0 AND security_status='pass')::int,
|
|
||||||
'skillmd_q', COUNT(*) FILTER (WHERE source_format='skill.md' AND LENGTH(description)>50 AND raw_content IS NOT NULL AND LENGTH(raw_content)>300 AND security_status='pass')::int
|
|
||||||
) AS data FROM active
|
|
||||||
),
|
|
||||||
repo_types AS (
|
|
||||||
SELECT json_build_object(
|
|
||||||
'single', COUNT(*) FILTER (WHERE cnt=1)::int, 'two', COUNT(*) FILTER (WHERE cnt=2)::int,
|
|
||||||
'multi', COUNT(*) FILTER (WHERE cnt>=3)::int, 'total', COUNT(*)::int
|
|
||||||
) AS data FROM (SELECT COUNT(*)::int AS cnt FROM active GROUP BY github_owner, github_repo) x
|
|
||||||
),
|
|
||||||
name_patterns AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT CASE
|
|
||||||
WHEN name ILIKE '%rule%' OR name ILIKE '%config%' OR name ILIKE '%setup%' THEN 'rules/config'
|
|
||||||
WHEN name ILIKE '%cursor%' OR name ILIKE '%claude%' OR name ILIKE '%copilot%' THEN 'tool-specific'
|
|
||||||
WHEN name ILIKE '%project%' OR name ILIKE '%team%' THEN 'project/team'
|
|
||||||
ELSE 'generic'
|
|
||||||
END AS pattern, COUNT(*)::int AS skills
|
|
||||||
FROM active GROUP BY 1 ORDER BY skills DESC
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
top_by_stars AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT id, name, github_stars AS stars, COALESCE(download_count,0) AS dl,
|
|
||||||
security_status AS sec, source_format AS fmt, LEFT(description,80) AS description
|
|
||||||
FROM active ORDER BY github_stars DESC LIMIT 15
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
discovered AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT discovered_via AS source, COUNT(*)::int AS total,
|
|
||||||
COUNT(*) FILTER (WHERE has_skill_md)::int AS with_skills
|
|
||||||
FROM discovered_repos GROUP BY discovered_via ORDER BY total DESC
|
|
||||||
) t
|
|
||||||
),
|
|
||||||
hash_dupes AS (
|
|
||||||
SELECT json_agg(row_to_json(t)) AS data FROM (
|
|
||||||
SELECT content_hash, COUNT(*)::int AS copies, MIN(name) AS sample
|
|
||||||
FROM active WHERE content_hash IS NOT NULL
|
|
||||||
GROUP BY content_hash HAVING COUNT(*)>2 ORDER BY copies DESC LIMIT 15
|
|
||||||
) t
|
|
||||||
)
|
|
||||||
SELECT json_build_object(
|
|
||||||
'overall', (SELECT data FROM overall),
|
|
||||||
'sourceFormats', (SELECT data FROM source_formats),
|
|
||||||
'starsDist', (SELECT data FROM stars_dist),
|
|
||||||
'dlDist', (SELECT data FROM dl_dist),
|
|
||||||
'topDL', (SELECT data FROM top_dl),
|
|
||||||
'topVW', (SELECT data FROM top_vw),
|
|
||||||
'security', (SELECT data FROM security),
|
|
||||||
'contentStats', (SELECT data FROM content_stats),
|
|
||||||
'contentLen', (SELECT data FROM content_len),
|
|
||||||
'triggerStats', (SELECT data FROM trigger_stats),
|
|
||||||
'freshness', (SELECT data FROM freshness),
|
|
||||||
'topOwners', (SELECT data FROM top_owners),
|
|
||||||
'ownerConcentration', (SELECT data FROM owner_concentration),
|
|
||||||
'aggregators', (SELECT data FROM aggregators),
|
|
||||||
'collections', (SELECT data FROM collections),
|
|
||||||
'skillsPerRepo', (SELECT data FROM skills_per_repo),
|
|
||||||
'categories', (SELECT data FROM categories_dist),
|
|
||||||
'flags', (SELECT data FROM flags),
|
|
||||||
'crossStats', (SELECT data FROM cross_stats),
|
|
||||||
'usability', (SELECT data FROM usability),
|
|
||||||
'repoTypes', (SELECT data FROM repo_types),
|
|
||||||
'namePatterns', (SELECT data FROM name_patterns),
|
|
||||||
'topByStars', (SELECT data FROM top_by_stars),
|
|
||||||
'discovered', (SELECT data FROM discovered),
|
|
||||||
'hashDupes', (SELECT data FROM hash_dupes)
|
|
||||||
) AS report;
|
|
||||||
@@ -1,894 +0,0 @@
|
|||||||
#!/usr/bin/env tsx
|
|
||||||
/**
|
|
||||||
* Phase 1: Database Exploration for Curation
|
|
||||||
*
|
|
||||||
* Comprehensive analysis of all indexed skills to understand:
|
|
||||||
* - Overall statistics and distribution
|
|
||||||
* - Quality signals and content analysis
|
|
||||||
* - Owner/repo concentration
|
|
||||||
* - Freshness and activity
|
|
||||||
* - Usability signals for curation decisions
|
|
||||||
*
|
|
||||||
* Usage:
|
|
||||||
* cd services/indexer && npx tsx ../../scripts/curation/explore.ts
|
|
||||||
*
|
|
||||||
* Output: prints report to console + saves JSON to scripts/curation/explore-report.json
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { createDb, closeDb, sql } from '@skillhub/db';
|
|
||||||
import { writeFileSync } from 'fs';
|
|
||||||
import { resolve, dirname } from 'path';
|
|
||||||
import { fileURLToPath } from 'url';
|
|
||||||
|
|
||||||
// ─── Connection ────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
const db = createDb();
|
|
||||||
|
|
||||||
// ─── Helpers ───────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
function header(title: string) {
|
|
||||||
const line = '='.repeat(70);
|
|
||||||
console.log(`\n${line}`);
|
|
||||||
console.log(` ${title}`);
|
|
||||||
console.log(line);
|
|
||||||
}
|
|
||||||
|
|
||||||
function subHeader(title: string) {
|
|
||||||
console.log(`\n -- ${title} ${'─'.repeat(Math.max(0, 60 - title.length))}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
function table(rows: Record<string, unknown>[], maxRows = 30) {
|
|
||||||
if (rows.length === 0) {
|
|
||||||
console.log(' (no data)');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const display = rows.slice(0, maxRows);
|
|
||||||
const keys = Object.keys(display[0]);
|
|
||||||
const widths = keys.map(k =>
|
|
||||||
Math.max(k.length, ...display.map(r => String(r[k] ?? '').length))
|
|
||||||
);
|
|
||||||
|
|
||||||
// Header
|
|
||||||
console.log(' ' + keys.map((k, i) => k.padEnd(widths[i])).join(' '));
|
|
||||||
console.log(' ' + widths.map(w => '-'.repeat(w)).join(' '));
|
|
||||||
|
|
||||||
// Rows
|
|
||||||
for (const row of display) {
|
|
||||||
console.log(' ' + keys.map((k, i) => String(row[k] ?? '').padEnd(widths[i])).join(' '));
|
|
||||||
}
|
|
||||||
if (rows.length > maxRows) {
|
|
||||||
console.log(` ... and ${rows.length - maxRows} more rows`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function num(n: unknown): string {
|
|
||||||
return Number(n ?? 0).toLocaleString('en-US');
|
|
||||||
}
|
|
||||||
|
|
||||||
function pct(part: unknown, total: unknown): string {
|
|
||||||
const p = Number(part ?? 0);
|
|
||||||
const t = Number(total ?? 1);
|
|
||||||
if (t === 0) return '0.0%';
|
|
||||||
return (p / t * 100).toFixed(1) + '%';
|
|
||||||
}
|
|
||||||
|
|
||||||
// Helper: execute raw SQL and return rows
|
|
||||||
async function query<T = Record<string, unknown>>(strings: TemplateStringsArray, ...values: unknown[]): Promise<T[]> {
|
|
||||||
const result = await db.execute(sql.raw(String.raw(strings, ...values)));
|
|
||||||
return result.rows as T[];
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Report accumulator ────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
const report: Record<string, unknown> = {};
|
|
||||||
|
|
||||||
// ─── Queries ───────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
async function runExploration() {
|
|
||||||
const startTime = Date.now();
|
|
||||||
|
|
||||||
// ======================================================================
|
|
||||||
// 1. OVERALL STATISTICS
|
|
||||||
// ======================================================================
|
|
||||||
header('1. OVERALL STATISTICS');
|
|
||||||
|
|
||||||
const overallRows = await db.execute(sql`
|
|
||||||
SELECT
|
|
||||||
COUNT(*)::int AS total_skills,
|
|
||||||
COUNT(*) FILTER (WHERE is_blocked = false)::int AS active_skills,
|
|
||||||
COUNT(*) FILTER (WHERE is_blocked = true)::int AS blocked_skills,
|
|
||||||
COUNT(DISTINCT github_owner)::int AS unique_owners,
|
|
||||||
COUNT(DISTINCT github_owner || '/' || github_repo)::int AS unique_repos,
|
|
||||||
COUNT(*) FILTER (WHERE source_format = 'skill.md')::int AS skill_md_count,
|
|
||||||
COUNT(*) FILTER (WHERE source_format != 'skill.md')::int AS other_format_count,
|
|
||||||
COALESCE(SUM(download_count), 0)::bigint AS total_downloads,
|
|
||||||
COALESCE(SUM(view_count), 0)::bigint AS total_views,
|
|
||||||
COUNT(*) FILTER (WHERE rating_count > 0)::int AS rated_skills,
|
|
||||||
COALESCE(SUM(rating_count), 0)::int AS total_ratings,
|
|
||||||
MIN(created_at)::text AS earliest_skill,
|
|
||||||
MAX(created_at)::text AS latest_skill
|
|
||||||
FROM skills
|
|
||||||
`);
|
|
||||||
const overall = overallRows.rows[0] as Record<string, unknown>;
|
|
||||||
report.overall = overall;
|
|
||||||
|
|
||||||
console.log(` Total skills: ${num(overall.total_skills)}`);
|
|
||||||
console.log(` Active (not blocked): ${num(overall.active_skills)}`);
|
|
||||||
console.log(` Blocked: ${num(overall.blocked_skills)}`);
|
|
||||||
console.log(` Unique owners: ${num(overall.unique_owners)}`);
|
|
||||||
console.log(` Unique repos: ${num(overall.unique_repos)}`);
|
|
||||||
console.log(` SKILL.md format: ${num(overall.skill_md_count)} (${pct(overall.skill_md_count, overall.total_skills)})`);
|
|
||||||
console.log(` Other formats: ${num(overall.other_format_count)} (${pct(overall.other_format_count, overall.total_skills)})`);
|
|
||||||
console.log(` Total downloads: ${num(overall.total_downloads)}`);
|
|
||||||
console.log(` Total views: ${num(overall.total_views)}`);
|
|
||||||
console.log(` Rated skills: ${num(overall.rated_skills)}`);
|
|
||||||
console.log(` Total ratings: ${num(overall.total_ratings)}`);
|
|
||||||
console.log(` Date range: ${overall.earliest_skill} -> ${overall.latest_skill}`);
|
|
||||||
|
|
||||||
const total = Number(overall.active_skills);
|
|
||||||
|
|
||||||
// Source format distribution
|
|
||||||
subHeader('Source Format Distribution');
|
|
||||||
const sfRows = await db.execute(sql`
|
|
||||||
SELECT
|
|
||||||
COALESCE(source_format, 'null') AS format,
|
|
||||||
COUNT(*)::int AS count,
|
|
||||||
COUNT(*) FILTER (WHERE is_blocked = false)::int AS active
|
|
||||||
FROM skills
|
|
||||||
GROUP BY source_format
|
|
||||||
ORDER BY count DESC
|
|
||||||
`);
|
|
||||||
report.sourceFormats = sfRows.rows;
|
|
||||||
table(sfRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
// ======================================================================
|
|
||||||
// 2. STARS DISTRIBUTION
|
|
||||||
// ======================================================================
|
|
||||||
header('2. GITHUB STARS DISTRIBUTION');
|
|
||||||
|
|
||||||
const starsRows = await db.execute(sql`
|
|
||||||
SELECT
|
|
||||||
CASE
|
|
||||||
WHEN github_stars IS NULL OR github_stars = 0 THEN '0 stars'
|
|
||||||
WHEN github_stars BETWEEN 1 AND 5 THEN '1-5'
|
|
||||||
WHEN github_stars BETWEEN 6 AND 10 THEN '6-10'
|
|
||||||
WHEN github_stars BETWEEN 11 AND 50 THEN '11-50'
|
|
||||||
WHEN github_stars BETWEEN 51 AND 100 THEN '51-100'
|
|
||||||
WHEN github_stars BETWEEN 101 AND 500 THEN '101-500'
|
|
||||||
WHEN github_stars BETWEEN 501 AND 1000 THEN '501-1K'
|
|
||||||
WHEN github_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
|
|
||||||
WHEN github_stars BETWEEN 5001 AND 10000 THEN '5K-10K'
|
|
||||||
WHEN github_stars BETWEEN 10001 AND 50000 THEN '10K-50K'
|
|
||||||
ELSE '50K+'
|
|
||||||
END AS star_range,
|
|
||||||
COUNT(*)::int AS skills,
|
|
||||||
COUNT(DISTINCT github_owner)::int AS owners,
|
|
||||||
COUNT(DISTINCT github_owner || '/' || github_repo)::int AS repos
|
|
||||||
FROM skills
|
|
||||||
WHERE is_blocked = false
|
|
||||||
GROUP BY
|
|
||||||
CASE
|
|
||||||
WHEN github_stars IS NULL OR github_stars = 0 THEN 0
|
|
||||||
WHEN github_stars BETWEEN 1 AND 5 THEN 1
|
|
||||||
WHEN github_stars BETWEEN 6 AND 10 THEN 2
|
|
||||||
WHEN github_stars BETWEEN 11 AND 50 THEN 3
|
|
||||||
WHEN github_stars BETWEEN 51 AND 100 THEN 4
|
|
||||||
WHEN github_stars BETWEEN 101 AND 500 THEN 5
|
|
||||||
WHEN github_stars BETWEEN 501 AND 1000 THEN 6
|
|
||||||
WHEN github_stars BETWEEN 1001 AND 5000 THEN 7
|
|
||||||
WHEN github_stars BETWEEN 5001 AND 10000 THEN 8
|
|
||||||
WHEN github_stars BETWEEN 10001 AND 50000 THEN 9
|
|
||||||
ELSE 10
|
|
||||||
END,
|
|
||||||
CASE
|
|
||||||
WHEN github_stars IS NULL OR github_stars = 0 THEN '0 stars'
|
|
||||||
WHEN github_stars BETWEEN 1 AND 5 THEN '1-5'
|
|
||||||
WHEN github_stars BETWEEN 6 AND 10 THEN '6-10'
|
|
||||||
WHEN github_stars BETWEEN 11 AND 50 THEN '11-50'
|
|
||||||
WHEN github_stars BETWEEN 51 AND 100 THEN '51-100'
|
|
||||||
WHEN github_stars BETWEEN 101 AND 500 THEN '101-500'
|
|
||||||
WHEN github_stars BETWEEN 501 AND 1000 THEN '501-1K'
|
|
||||||
WHEN github_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
|
|
||||||
WHEN github_stars BETWEEN 5001 AND 10000 THEN '5K-10K'
|
|
||||||
WHEN github_stars BETWEEN 10001 AND 50000 THEN '10K-50K'
|
|
||||||
ELSE '50K+'
|
|
||||||
END
|
|
||||||
ORDER BY MIN(COALESCE(github_stars, 0))
|
|
||||||
`);
|
|
||||||
report.starsDist = starsRows.rows;
|
|
||||||
table(starsRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
// ======================================================================
|
|
||||||
// 3. ENGAGEMENT DISTRIBUTION
|
|
||||||
// ======================================================================
|
|
||||||
header('3. ENGAGEMENT (Downloads & Views)');
|
|
||||||
|
|
||||||
subHeader('Download Distribution');
|
|
||||||
const dlRows = await db.execute(sql`
|
|
||||||
SELECT
|
|
||||||
CASE
|
|
||||||
WHEN COALESCE(download_count, 0) = 0 THEN '0'
|
|
||||||
WHEN download_count BETWEEN 1 AND 5 THEN '1-5'
|
|
||||||
WHEN download_count BETWEEN 6 AND 10 THEN '6-10'
|
|
||||||
WHEN download_count BETWEEN 11 AND 50 THEN '11-50'
|
|
||||||
WHEN download_count BETWEEN 51 AND 100 THEN '51-100'
|
|
||||||
WHEN download_count BETWEEN 101 AND 500 THEN '101-500'
|
|
||||||
ELSE '500+'
|
|
||||||
END AS range,
|
|
||||||
COUNT(*)::int AS skills
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY 1 ORDER BY MIN(COALESCE(download_count, 0))
|
|
||||||
`);
|
|
||||||
report.downloadDist = dlRows.rows;
|
|
||||||
table(dlRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
subHeader('View Distribution');
|
|
||||||
const vwRows = await db.execute(sql`
|
|
||||||
SELECT
|
|
||||||
CASE
|
|
||||||
WHEN COALESCE(view_count, 0) = 0 THEN '0'
|
|
||||||
WHEN view_count BETWEEN 1 AND 10 THEN '1-10'
|
|
||||||
WHEN view_count BETWEEN 11 AND 50 THEN '11-50'
|
|
||||||
WHEN view_count BETWEEN 51 AND 100 THEN '51-100'
|
|
||||||
WHEN view_count BETWEEN 101 AND 500 THEN '101-500'
|
|
||||||
WHEN view_count BETWEEN 501 AND 1000 THEN '501-1K'
|
|
||||||
ELSE '1K+'
|
|
||||||
END AS range,
|
|
||||||
COUNT(*)::int AS skills
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY 1 ORDER BY MIN(COALESCE(view_count, 0))
|
|
||||||
`);
|
|
||||||
report.viewDist = vwRows.rows;
|
|
||||||
table(vwRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
subHeader('Top 20 Downloaded Skills');
|
|
||||||
const topDL = await db.execute(sql`
|
|
||||||
SELECT id, name, download_count AS downloads, view_count AS views,
|
|
||||||
github_stars AS stars, github_owner AS owner
|
|
||||||
FROM skills
|
|
||||||
WHERE is_blocked = false AND COALESCE(download_count, 0) > 0
|
|
||||||
ORDER BY download_count DESC LIMIT 20
|
|
||||||
`);
|
|
||||||
report.topDownloads = topDL.rows;
|
|
||||||
table(topDL.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
subHeader('Top 20 Viewed Skills');
|
|
||||||
const topVW = await db.execute(sql`
|
|
||||||
SELECT id, name, view_count AS views, download_count AS downloads,
|
|
||||||
github_stars AS stars, github_owner AS owner
|
|
||||||
FROM skills
|
|
||||||
WHERE is_blocked = false AND COALESCE(view_count, 0) > 0
|
|
||||||
ORDER BY view_count DESC LIMIT 20
|
|
||||||
`);
|
|
||||||
report.topViews = topVW.rows;
|
|
||||||
table(topVW.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
// ======================================================================
|
|
||||||
// 4. SECURITY STATUS
|
|
||||||
// ======================================================================
|
|
||||||
header('4. SECURITY STATUS');
|
|
||||||
|
|
||||||
const secRows = await db.execute(sql`
|
|
||||||
SELECT
|
|
||||||
COALESCE(security_status, 'null') AS status,
|
|
||||||
COUNT(*)::int AS count,
|
|
||||||
ROUND(COUNT(*)::numeric / NULLIF(${total}, 0) * 100, 1) AS pct
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY security_status ORDER BY count DESC
|
|
||||||
`);
|
|
||||||
report.securityDist = secRows.rows;
|
|
||||||
table(secRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
// ======================================================================
|
|
||||||
// 5. CONTENT ANALYSIS
|
|
||||||
// ======================================================================
|
|
||||||
header('5. CONTENT ANALYSIS');
|
|
||||||
|
|
||||||
const csRows = await db.execute(sql`
|
|
||||||
SELECT
|
|
||||||
COUNT(*) FILTER (WHERE raw_content IS NOT NULL)::int AS has_content,
|
|
||||||
COUNT(*) FILTER (WHERE raw_content IS NULL)::int AS no_content,
|
|
||||||
ROUND(AVG(LENGTH(raw_content)) FILTER (WHERE raw_content IS NOT NULL))::int AS avg_content_len,
|
|
||||||
MAX(LENGTH(raw_content))::int AS max_content_len,
|
|
||||||
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY LENGTH(raw_content))
|
|
||||||
FILTER (WHERE raw_content IS NOT NULL)::int AS median_content_len,
|
|
||||||
ROUND(AVG(LENGTH(description)))::int AS avg_desc_len,
|
|
||||||
COUNT(*) FILTER (WHERE LENGTH(description) > 100)::int AS good_desc,
|
|
||||||
COUNT(*) FILTER (WHERE LENGTH(description) <= 20)::int AS short_desc,
|
|
||||||
COUNT(*) FILTER (WHERE version IS NOT NULL)::int AS has_version,
|
|
||||||
COUNT(*) FILTER (WHERE license IS NOT NULL)::int AS has_license,
|
|
||||||
COUNT(*) FILTER (WHERE author IS NOT NULL)::int AS has_author,
|
|
||||||
COUNT(*) FILTER (WHERE homepage IS NOT NULL)::int AS has_homepage
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
`);
|
|
||||||
const cs = csRows.rows[0] as Record<string, unknown>;
|
|
||||||
report.contentStats = cs;
|
|
||||||
|
|
||||||
console.log(` Has raw_content: ${num(cs.has_content)} (${pct(cs.has_content, total)})`);
|
|
||||||
console.log(` No raw_content: ${num(cs.no_content)} (${pct(cs.no_content, total)})`);
|
|
||||||
console.log(` Avg content length: ${num(cs.avg_content_len)} chars`);
|
|
||||||
console.log(` Median content len: ${num(cs.median_content_len)} chars`);
|
|
||||||
console.log(` Max content length: ${num(cs.max_content_len)} chars`);
|
|
||||||
console.log(` Avg desc length: ${num(cs.avg_desc_len)} chars`);
|
|
||||||
console.log(` Good desc (>100ch): ${num(cs.good_desc)} (${pct(cs.good_desc, total)})`);
|
|
||||||
console.log(` Short desc (<=20): ${num(cs.short_desc)} (${pct(cs.short_desc, total)})`);
|
|
||||||
console.log(` Has version: ${num(cs.has_version)} (${pct(cs.has_version, total)})`);
|
|
||||||
console.log(` Has license: ${num(cs.has_license)} (${pct(cs.has_license, total)})`);
|
|
||||||
console.log(` Has author: ${num(cs.has_author)} (${pct(cs.has_author, total)})`);
|
|
||||||
console.log(` Has homepage: ${num(cs.has_homepage)} (${pct(cs.has_homepage, total)})`);
|
|
||||||
|
|
||||||
subHeader('Content Length Distribution');
|
|
||||||
const clRows = await db.execute(sql`
|
|
||||||
SELECT
|
|
||||||
CASE
|
|
||||||
WHEN raw_content IS NULL THEN 'null'
|
|
||||||
WHEN LENGTH(raw_content) = 0 THEN 'empty'
|
|
||||||
WHEN LENGTH(raw_content) < 200 THEN '<200'
|
|
||||||
WHEN LENGTH(raw_content) < 500 THEN '200-500'
|
|
||||||
WHEN LENGTH(raw_content) < 1000 THEN '500-1K'
|
|
||||||
WHEN LENGTH(raw_content) < 3000 THEN '1K-3K'
|
|
||||||
WHEN LENGTH(raw_content) < 5000 THEN '3K-5K'
|
|
||||||
WHEN LENGTH(raw_content) < 10000 THEN '5K-10K'
|
|
||||||
WHEN LENGTH(raw_content) < 50000 THEN '10K-50K'
|
|
||||||
ELSE '50K+'
|
|
||||||
END AS range,
|
|
||||||
COUNT(*)::int AS skills
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY 1 ORDER BY MIN(COALESCE(LENGTH(raw_content), -1))
|
|
||||||
`);
|
|
||||||
report.contentLenDist = clRows.rows;
|
|
||||||
table(clRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
// ======================================================================
|
|
||||||
// 6. TRIGGERS & COMPATIBILITY
|
|
||||||
// ======================================================================
|
|
||||||
header('6. TRIGGERS & COMPATIBILITY');
|
|
||||||
|
|
||||||
const trRows = await db.execute(sql`
|
|
||||||
SELECT
|
|
||||||
COUNT(*) FILTER (WHERE triggers IS NOT NULL)::int AS has_triggers,
|
|
||||||
COUNT(*) FILTER (WHERE triggers IS NULL)::int AS no_triggers,
|
|
||||||
COUNT(*) FILTER (WHERE triggers->>'filePatterns' IS NOT NULL
|
|
||||||
AND triggers->>'filePatterns' != '[]'
|
|
||||||
AND triggers->>'filePatterns' != 'null')::int AS has_file_patterns,
|
|
||||||
COUNT(*) FILTER (WHERE triggers->>'keywords' IS NOT NULL
|
|
||||||
AND triggers->>'keywords' != '[]'
|
|
||||||
AND triggers->>'keywords' != 'null')::int AS has_keywords,
|
|
||||||
COUNT(*) FILTER (WHERE triggers->>'languages' IS NOT NULL
|
|
||||||
AND triggers->>'languages' != '[]'
|
|
||||||
AND triggers->>'languages' != 'null')::int AS has_languages,
|
|
||||||
COUNT(*) FILTER (WHERE compatibility IS NOT NULL)::int AS has_compat,
|
|
||||||
COUNT(*) FILTER (WHERE compatibility->>'platforms' IS NOT NULL
|
|
||||||
AND compatibility->>'platforms' != '[]'
|
|
||||||
AND compatibility->>'platforms' != 'null')::int AS has_platforms,
|
|
||||||
COUNT(*) FILTER (WHERE compatibility->>'requires' IS NOT NULL
|
|
||||||
AND compatibility->>'requires' != '[]'
|
|
||||||
AND compatibility->>'requires' != 'null')::int AS has_requires
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
`);
|
|
||||||
const tr = trRows.rows[0] as Record<string, unknown>;
|
|
||||||
report.triggerStats = tr;
|
|
||||||
|
|
||||||
console.log(` Has triggers: ${num(tr.has_triggers)} (${pct(tr.has_triggers, total)})`);
|
|
||||||
console.log(` No triggers: ${num(tr.no_triggers)} (${pct(tr.no_triggers, total)})`);
|
|
||||||
console.log(` - filePatterns: ${num(tr.has_file_patterns)}`);
|
|
||||||
console.log(` - keywords: ${num(tr.has_keywords)}`);
|
|
||||||
console.log(` - languages: ${num(tr.has_languages)}`);
|
|
||||||
console.log(` Has compatibility: ${num(tr.has_compat)}`);
|
|
||||||
console.log(` - platforms: ${num(tr.has_platforms)}`);
|
|
||||||
console.log(` - requires: ${num(tr.has_requires)}`);
|
|
||||||
|
|
||||||
// ======================================================================
|
|
||||||
// 7. FRESHNESS
|
|
||||||
// ======================================================================
|
|
||||||
header('7. FRESHNESS & ACTIVITY');
|
|
||||||
|
|
||||||
subHeader('Created At Distribution');
|
|
||||||
const crRows = await db.execute(sql`
|
|
||||||
SELECT
|
|
||||||
CASE
|
|
||||||
WHEN created_at > NOW() - INTERVAL '7 days' THEN 'Last 7d'
|
|
||||||
WHEN created_at > NOW() - INTERVAL '30 days' THEN 'Last 30d'
|
|
||||||
WHEN created_at > NOW() - INTERVAL '90 days' THEN 'Last 90d'
|
|
||||||
WHEN created_at > NOW() - INTERVAL '180 days' THEN 'Last 180d'
|
|
||||||
WHEN created_at > NOW() - INTERVAL '365 days' THEN 'Last 365d'
|
|
||||||
ELSE 'Older'
|
|
||||||
END AS range,
|
|
||||||
COUNT(*)::int AS skills
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY 1 ORDER BY MIN(NOW() - created_at)
|
|
||||||
`);
|
|
||||||
report.createdDist = crRows.rows;
|
|
||||||
table(crRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
subHeader('Updated At Distribution');
|
|
||||||
const upRows = await db.execute(sql`
|
|
||||||
SELECT
|
|
||||||
CASE
|
|
||||||
WHEN updated_at > NOW() - INTERVAL '7 days' THEN 'Last 7d'
|
|
||||||
WHEN updated_at > NOW() - INTERVAL '30 days' THEN 'Last 30d'
|
|
||||||
WHEN updated_at > NOW() - INTERVAL '90 days' THEN 'Last 90d'
|
|
||||||
WHEN updated_at > NOW() - INTERVAL '180 days' THEN 'Last 180d'
|
|
||||||
WHEN updated_at > NOW() - INTERVAL '365 days' THEN 'Last 365d'
|
|
||||||
ELSE 'Older'
|
|
||||||
END AS range,
|
|
||||||
COUNT(*)::int AS skills
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY 1 ORDER BY MIN(NOW() - updated_at)
|
|
||||||
`);
|
|
||||||
report.updatedDist = upRows.rows;
|
|
||||||
table(upRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
subHeader('Last Downloaded Distribution');
|
|
||||||
const ldRows = await db.execute(sql`
|
|
||||||
SELECT
|
|
||||||
CASE
|
|
||||||
WHEN last_downloaded_at IS NULL THEN 'Never'
|
|
||||||
WHEN last_downloaded_at > NOW() - INTERVAL '7 days' THEN 'Last 7d'
|
|
||||||
WHEN last_downloaded_at > NOW() - INTERVAL '30 days' THEN 'Last 30d'
|
|
||||||
WHEN last_downloaded_at > NOW() - INTERVAL '90 days' THEN 'Last 90d'
|
|
||||||
ELSE 'Older'
|
|
||||||
END AS range,
|
|
||||||
COUNT(*)::int AS skills
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY 1 ORDER BY MIN(COALESCE(last_downloaded_at, '1970-01-01'::timestamp))
|
|
||||||
`);
|
|
||||||
report.lastDownDist = ldRows.rows;
|
|
||||||
table(ldRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
// ======================================================================
|
|
||||||
// 8. TOP OWNERS
|
|
||||||
// ======================================================================
|
|
||||||
header('8. TOP OWNERS');
|
|
||||||
|
|
||||||
subHeader('Top 30 Owners by Skill Count');
|
|
||||||
const owRows = await db.execute(sql`
|
|
||||||
SELECT
|
|
||||||
github_owner AS owner,
|
|
||||||
COUNT(*)::int AS skills,
|
|
||||||
COUNT(DISTINCT github_repo)::int AS repos,
|
|
||||||
MAX(github_stars)::int AS max_stars,
|
|
||||||
COALESCE(SUM(download_count), 0)::int AS downloads,
|
|
||||||
COALESCE(SUM(view_count), 0)::int AS views
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY github_owner ORDER BY skills DESC LIMIT 30
|
|
||||||
`);
|
|
||||||
report.topOwnersByCount = owRows.rows;
|
|
||||||
table(owRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
subHeader('Top 20 Owners by Stars');
|
|
||||||
const osRows = await db.execute(sql`
|
|
||||||
SELECT
|
|
||||||
github_owner AS owner,
|
|
||||||
MAX(github_stars)::int AS max_stars,
|
|
||||||
COUNT(*)::int AS skills,
|
|
||||||
COUNT(DISTINCT github_repo)::int AS repos,
|
|
||||||
COALESCE(SUM(download_count), 0)::int AS downloads
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY github_owner ORDER BY max_stars DESC LIMIT 20
|
|
||||||
`);
|
|
||||||
report.topOwnersByStars = osRows.rows;
|
|
||||||
table(osRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
subHeader('Owner Concentration');
|
|
||||||
const ocRows = await db.execute(sql`
|
|
||||||
WITH ranked AS (
|
|
||||||
SELECT github_owner, COUNT(*)::int AS cnt,
|
|
||||||
ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY github_owner
|
|
||||||
)
|
|
||||||
SELECT
|
|
||||||
SUM(cnt) FILTER (WHERE rn <= 1)::int AS top_1,
|
|
||||||
SUM(cnt) FILTER (WHERE rn <= 5)::int AS top_5,
|
|
||||||
SUM(cnt) FILTER (WHERE rn <= 10)::int AS top_10,
|
|
||||||
SUM(cnt) FILTER (WHERE rn <= 20)::int AS top_20,
|
|
||||||
SUM(cnt) FILTER (WHERE rn <= 50)::int AS top_50,
|
|
||||||
SUM(cnt)::int AS total,
|
|
||||||
COUNT(*)::int AS owners
|
|
||||||
FROM ranked
|
|
||||||
`);
|
|
||||||
const oc = ocRows.rows[0] as Record<string, unknown>;
|
|
||||||
report.ownerConcentration = oc;
|
|
||||||
|
|
||||||
console.log(` Top 1 owner: ${num(oc.top_1)} skills (${pct(oc.top_1, oc.total)})`);
|
|
||||||
console.log(` Top 5 owners: ${num(oc.top_5)} skills (${pct(oc.top_5, oc.total)})`);
|
|
||||||
console.log(` Top 10 owners: ${num(oc.top_10)} skills (${pct(oc.top_10, oc.total)})`);
|
|
||||||
console.log(` Top 20 owners: ${num(oc.top_20)} skills (${pct(oc.top_20, oc.total)})`);
|
|
||||||
console.log(` Top 50 owners: ${num(oc.top_50)} skills (${pct(oc.top_50, oc.total)})`);
|
|
||||||
console.log(` Total owners: ${num(oc.owners)}`);
|
|
||||||
|
|
||||||
// ======================================================================
|
|
||||||
// 9. MULTI-SKILL REPOS
|
|
||||||
// ======================================================================
|
|
||||||
header('9. MULTI-SKILL REPOS');
|
|
||||||
|
|
||||||
subHeader('Repos with 20+ Skills (Aggregator Candidates)');
|
|
||||||
const aggRows = await db.execute(sql`
|
|
||||||
SELECT
|
|
||||||
github_owner || '/' || github_repo AS repo,
|
|
||||||
COUNT(*)::int AS skills,
|
|
||||||
MAX(github_stars)::int AS stars,
|
|
||||||
COALESCE(SUM(download_count), 0)::int AS downloads
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY github_owner, github_repo
|
|
||||||
HAVING COUNT(*) >= 20
|
|
||||||
ORDER BY skills DESC LIMIT 30
|
|
||||||
`);
|
|
||||||
report.aggregatorCandidates = aggRows.rows;
|
|
||||||
table(aggRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
subHeader('Repos with 3-19 Skills (Collection Candidates)');
|
|
||||||
const colRows = await db.execute(sql`
|
|
||||||
SELECT
|
|
||||||
github_owner || '/' || github_repo AS repo,
|
|
||||||
COUNT(*)::int AS skills,
|
|
||||||
MAX(github_stars)::int AS stars,
|
|
||||||
COALESCE(SUM(download_count), 0)::int AS downloads
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY github_owner, github_repo
|
|
||||||
HAVING COUNT(*) BETWEEN 3 AND 19
|
|
||||||
ORDER BY skills DESC LIMIT 30
|
|
||||||
`);
|
|
||||||
report.collectionCandidates = colRows.rows;
|
|
||||||
table(colRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
subHeader('Skills-per-Repo Distribution');
|
|
||||||
const sprRows = await db.execute(sql`
|
|
||||||
WITH rc AS (
|
|
||||||
SELECT COUNT(*)::int AS cnt
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY github_owner, github_repo
|
|
||||||
)
|
|
||||||
SELECT
|
|
||||||
CASE
|
|
||||||
WHEN cnt = 1 THEN '1 skill'
|
|
||||||
WHEN cnt = 2 THEN '2 skills'
|
|
||||||
WHEN cnt BETWEEN 3 AND 5 THEN '3-5'
|
|
||||||
WHEN cnt BETWEEN 6 AND 10 THEN '6-10'
|
|
||||||
WHEN cnt BETWEEN 11 AND 20 THEN '11-20'
|
|
||||||
WHEN cnt BETWEEN 21 AND 50 THEN '21-50'
|
|
||||||
WHEN cnt BETWEEN 51 AND 100 THEN '51-100'
|
|
||||||
ELSE '100+'
|
|
||||||
END AS per_repo,
|
|
||||||
COUNT(*)::int AS repos,
|
|
||||||
SUM(cnt)::int AS total_skills
|
|
||||||
FROM rc
|
|
||||||
GROUP BY 1 ORDER BY MIN(cnt)
|
|
||||||
`);
|
|
||||||
report.skillsPerRepo = sprRows.rows;
|
|
||||||
table(sprRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
// ======================================================================
|
|
||||||
// 10. CATEGORY DISTRIBUTION
|
|
||||||
// ======================================================================
|
|
||||||
header('10. CATEGORY DISTRIBUTION');
|
|
||||||
|
|
||||||
const catRows = await db.execute(sql`
|
|
||||||
SELECT c.name AS category, c.skill_count::int AS skills, c.id
|
|
||||||
FROM categories c
|
|
||||||
WHERE c.id NOT LIKE 'parent-%'
|
|
||||||
ORDER BY c.skill_count DESC
|
|
||||||
`);
|
|
||||||
report.categoryDist = catRows.rows;
|
|
||||||
table(catRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
const ncRows = await db.execute(sql`
|
|
||||||
SELECT COUNT(*)::int AS count
|
|
||||||
FROM skills s
|
|
||||||
WHERE s.is_blocked = false
|
|
||||||
AND NOT EXISTS (SELECT 1 FROM skill_categories sc WHERE sc.skill_id = s.id)
|
|
||||||
`);
|
|
||||||
report.uncategorizedCount = (ncRows.rows[0] as Record<string, unknown>)?.count;
|
|
||||||
console.log(`\n Skills with no category: ${num(report.uncategorizedCount)}`);
|
|
||||||
|
|
||||||
// ======================================================================
|
|
||||||
// 11. FLAGS & RATINGS
|
|
||||||
// ======================================================================
|
|
||||||
header('11. FLAGS & RATINGS');
|
|
||||||
|
|
||||||
const flRows = await db.execute(sql`
|
|
||||||
SELECT
|
|
||||||
COUNT(*) FILTER (WHERE is_verified = true)::int AS verified,
|
|
||||||
COUNT(*) FILTER (WHERE is_featured = true)::int AS featured,
|
|
||||||
COUNT(*) FILTER (WHERE rating IS NOT NULL)::int AS has_rating,
|
|
||||||
COUNT(*) FILTER (WHERE rating_count > 0)::int AS has_user_ratings,
|
|
||||||
ROUND(AVG(rating) FILTER (WHERE rating IS NOT NULL), 2)::numeric AS avg_rating,
|
|
||||||
MAX(rating_count)::int AS max_rating_count
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
`);
|
|
||||||
const fl = flRows.rows[0] as Record<string, unknown>;
|
|
||||||
report.flags = fl;
|
|
||||||
|
|
||||||
console.log(` Verified: ${num(fl.verified)}`);
|
|
||||||
console.log(` Featured: ${num(fl.featured)}`);
|
|
||||||
console.log(` Has rating: ${num(fl.has_rating)}`);
|
|
||||||
console.log(` Has user ratings: ${num(fl.has_user_ratings)}`);
|
|
||||||
console.log(` Average rating: ${fl.avg_rating ?? 'N/A'}`);
|
|
||||||
console.log(` Max rating count: ${num(fl.max_rating_count)}`);
|
|
||||||
|
|
||||||
// ======================================================================
|
|
||||||
// 12. CROSS-TABLE STATS
|
|
||||||
// ======================================================================
|
|
||||||
header('12. CROSS-TABLE STATS');
|
|
||||||
|
|
||||||
const xtRows = await db.execute(sql`
|
|
||||||
SELECT
|
|
||||||
(SELECT COUNT(*)::int FROM users) AS users,
|
|
||||||
(SELECT COUNT(*)::int FROM ratings) AS ratings,
|
|
||||||
(SELECT COUNT(*)::int FROM installations) AS installations,
|
|
||||||
(SELECT COUNT(*)::int FROM favorites) AS favorites,
|
|
||||||
(SELECT COUNT(*)::int FROM discovered_repos) AS discovered_repos,
|
|
||||||
(SELECT COUNT(*) FILTER (WHERE has_skill_md = true)::int FROM discovered_repos) AS repos_with_skills,
|
|
||||||
(SELECT COUNT(*)::int FROM removal_requests) AS removal_requests,
|
|
||||||
(SELECT COUNT(*)::int FROM add_requests) AS add_requests,
|
|
||||||
(SELECT COUNT(*) FILTER (WHERE unsubscribed_at IS NULL)::int FROM email_subscriptions) AS subscribers
|
|
||||||
`);
|
|
||||||
const xt = xtRows.rows[0] as Record<string, unknown>;
|
|
||||||
report.crossStats = xt;
|
|
||||||
|
|
||||||
console.log(` Users: ${num(xt.users)}`);
|
|
||||||
console.log(` Ratings: ${num(xt.ratings)}`);
|
|
||||||
console.log(` Installations: ${num(xt.installations)}`);
|
|
||||||
console.log(` Favorites: ${num(xt.favorites)}`);
|
|
||||||
console.log(` Discovered repos: ${num(xt.discovered_repos)}`);
|
|
||||||
console.log(` - with skills: ${num(xt.repos_with_skills)}`);
|
|
||||||
console.log(` Removal requests: ${num(xt.removal_requests)}`);
|
|
||||||
console.log(` Add requests: ${num(xt.add_requests)}`);
|
|
||||||
console.log(` Subscribers: ${num(xt.subscribers)}`);
|
|
||||||
|
|
||||||
subHeader('Installations by Platform');
|
|
||||||
const ipRows = await db.execute(sql`
|
|
||||||
SELECT platform, COUNT(*)::int AS count
|
|
||||||
FROM installations GROUP BY platform ORDER BY count DESC
|
|
||||||
`);
|
|
||||||
report.installByPlatform = ipRows.rows;
|
|
||||||
table(ipRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
// ======================================================================
|
|
||||||
// 13. USABILITY SIGNALS
|
|
||||||
// ======================================================================
|
|
||||||
header('13. USABILITY SIGNALS FOR CURATION');
|
|
||||||
|
|
||||||
const usRows = await db.execute(sql`
|
|
||||||
SELECT
|
|
||||||
COUNT(*) FILTER (
|
|
||||||
WHERE LENGTH(description) > 50
|
|
||||||
AND raw_content IS NOT NULL AND LENGTH(raw_content) > 500
|
|
||||||
AND security_status = 'pass'
|
|
||||||
)::int AS strong_standalone,
|
|
||||||
|
|
||||||
COUNT(*) FILTER (
|
|
||||||
WHERE triggers IS NOT NULL AND triggers != '{}'::jsonb
|
|
||||||
AND (triggers->>'filePatterns' IS NOT NULL AND triggers->>'filePatterns' != '[]')
|
|
||||||
)::int AS has_file_triggers,
|
|
||||||
|
|
||||||
COUNT(*) FILTER (WHERE COALESCE(download_count, 0) > 0)::int AS ever_downloaded,
|
|
||||||
|
|
||||||
COUNT(*) FILTER (
|
|
||||||
WHERE github_stars >= 10
|
|
||||||
AND LENGTH(description) > 50
|
|
||||||
AND raw_content IS NOT NULL
|
|
||||||
AND security_status = 'pass'
|
|
||||||
)::int AS high_quality,
|
|
||||||
|
|
||||||
COUNT(*) FILTER (
|
|
||||||
WHERE github_stars >= 100
|
|
||||||
AND COALESCE(download_count, 0) > 0
|
|
||||||
AND security_status = 'pass'
|
|
||||||
)::int AS premium,
|
|
||||||
|
|
||||||
COUNT(*) FILTER (
|
|
||||||
WHERE source_format = 'skill.md'
|
|
||||||
AND LENGTH(description) > 50
|
|
||||||
AND raw_content IS NOT NULL AND LENGTH(raw_content) > 300
|
|
||||||
AND security_status = 'pass'
|
|
||||||
)::int AS skillmd_quality
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
`);
|
|
||||||
const us = usRows.rows[0] as Record<string, unknown>;
|
|
||||||
report.usability = us;
|
|
||||||
|
|
||||||
console.log(` Strong standalone candidates: ${num(us.strong_standalone)}`);
|
|
||||||
console.log(` (desc>50 + content>500 + security=pass)`);
|
|
||||||
console.log(` With file triggers: ${num(us.has_file_triggers)}`);
|
|
||||||
console.log(` Ever downloaded: ${num(us.ever_downloaded)}`);
|
|
||||||
console.log(` High quality (stars>10+desc+sec): ${num(us.high_quality)}`);
|
|
||||||
console.log(` Premium (stars>100+dl>0+sec): ${num(us.premium)}`);
|
|
||||||
console.log(` SKILL.md with quality: ${num(us.skillmd_quality)}`);
|
|
||||||
|
|
||||||
// ======================================================================
|
|
||||||
// 14. STANDALONE vs PROJECT-BOUND
|
|
||||||
// ======================================================================
|
|
||||||
header('14. STANDALONE vs PROJECT-BOUND HEURISTIC');
|
|
||||||
|
|
||||||
subHeader('Single vs Multi-Skill Repos');
|
|
||||||
const smRows = await db.execute(sql`
|
|
||||||
WITH rc AS (
|
|
||||||
SELECT github_owner, github_repo, COUNT(*)::int AS cnt
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY github_owner, github_repo
|
|
||||||
)
|
|
||||||
SELECT
|
|
||||||
COUNT(*) FILTER (WHERE cnt = 1)::int AS single,
|
|
||||||
COUNT(*) FILTER (WHERE cnt = 2)::int AS two,
|
|
||||||
COUNT(*) FILTER (WHERE cnt >= 3)::int AS multi,
|
|
||||||
COUNT(*)::int AS total
|
|
||||||
FROM rc
|
|
||||||
`);
|
|
||||||
const sm = smRows.rows[0] as Record<string, unknown>;
|
|
||||||
report.repoTypes = sm;
|
|
||||||
|
|
||||||
console.log(` Single-skill repos: ${num(sm.single)} (${pct(sm.single, sm.total)})`);
|
|
||||||
console.log(` Two-skill repos: ${num(sm.two)} (${pct(sm.two, sm.total)})`);
|
|
||||||
console.log(` Multi-skill repos: ${num(sm.multi)} (${pct(sm.multi, sm.total)})`);
|
|
||||||
|
|
||||||
subHeader('Name Pattern Signals');
|
|
||||||
const npRows = await db.execute(sql`
|
|
||||||
SELECT
|
|
||||||
CASE
|
|
||||||
WHEN name ILIKE '%rule%' OR name ILIKE '%config%' OR name ILIKE '%setup%' THEN 'rules/config/setup'
|
|
||||||
WHEN name ILIKE '%my-%' OR name ILIKE '%my_%' THEN 'starts with my-'
|
|
||||||
WHEN name ILIKE '%cursor%' OR name ILIKE '%claude%' OR name ILIKE '%copilot%' THEN 'tool-specific'
|
|
||||||
WHEN name ILIKE '%project%' OR name ILIKE '%team%' OR name ILIKE '%internal%' THEN 'project/team'
|
|
||||||
WHEN name ILIKE '%.mdc' OR name ILIKE '%cursorrule%' THEN 'cursor rules file'
|
|
||||||
ELSE 'generic/other'
|
|
||||||
END AS pattern,
|
|
||||||
COUNT(*)::int AS skills
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY 1 ORDER BY skills DESC
|
|
||||||
`);
|
|
||||||
report.namePatterns = npRows.rows;
|
|
||||||
table(npRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
// ======================================================================
|
|
||||||
// 15. SAMPLE SKILLS
|
|
||||||
// ======================================================================
|
|
||||||
header('15. SAMPLE SKILLS');
|
|
||||||
|
|
||||||
subHeader('Top 15 by Stars');
|
|
||||||
const tsRows = await db.execute(sql`
|
|
||||||
SELECT id, name, github_stars AS stars, COALESCE(download_count,0) AS dl,
|
|
||||||
security_status AS sec, source_format AS fmt,
|
|
||||||
LEFT(description, 80) AS description
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
ORDER BY github_stars DESC LIMIT 15
|
|
||||||
`);
|
|
||||||
report.topByStars = tsRows.rows;
|
|
||||||
table(tsRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
subHeader('Top 15 by Downloads (actually used)');
|
|
||||||
const tdRows = await db.execute(sql`
|
|
||||||
SELECT id, name, download_count AS dl, view_count AS views,
|
|
||||||
github_stars AS stars, security_status AS sec,
|
|
||||||
LEFT(description, 80) AS description
|
|
||||||
FROM skills WHERE is_blocked = false AND COALESCE(download_count,0) > 0
|
|
||||||
ORDER BY download_count DESC LIMIT 15
|
|
||||||
`);
|
|
||||||
report.topByDL = tdRows.rows;
|
|
||||||
table(tdRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
subHeader('SKILL.md + Stars>50 + Downloads>0');
|
|
||||||
const qsRows = await db.execute(sql`
|
|
||||||
SELECT id, name, github_stars AS stars, download_count AS dl,
|
|
||||||
security_status AS sec, LEFT(description, 80) AS description
|
|
||||||
FROM skills
|
|
||||||
WHERE is_blocked = false AND source_format = 'skill.md'
|
|
||||||
AND github_stars >= 50 AND COALESCE(download_count, 0) > 0
|
|
||||||
ORDER BY github_stars DESC LIMIT 20
|
|
||||||
`);
|
|
||||||
report.qualitySkillMd = qsRows.rows;
|
|
||||||
table(qsRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
// ======================================================================
|
|
||||||
// 16. DISCOVERED REPOS
|
|
||||||
// ======================================================================
|
|
||||||
header('16. DISCOVERED REPOS');
|
|
||||||
|
|
||||||
const drRows = await db.execute(sql`
|
|
||||||
SELECT
|
|
||||||
discovered_via AS source,
|
|
||||||
COUNT(*)::int AS total,
|
|
||||||
COUNT(*) FILTER (WHERE has_skill_md = true)::int AS with_skills,
|
|
||||||
COUNT(*) FILTER (WHERE last_scanned IS NOT NULL)::int AS scanned,
|
|
||||||
COUNT(*) FILTER (WHERE is_archived = true)::int AS archived
|
|
||||||
FROM discovered_repos
|
|
||||||
GROUP BY discovered_via ORDER BY total DESC
|
|
||||||
`);
|
|
||||||
report.discoveredStats = drRows.rows;
|
|
||||||
table(drRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
// ======================================================================
|
|
||||||
// 17. CACHED FILES
|
|
||||||
// ======================================================================
|
|
||||||
header('17. CACHED FILES');
|
|
||||||
|
|
||||||
const cfRows = await db.execute(sql`
|
|
||||||
SELECT
|
|
||||||
COUNT(*) FILTER (WHERE cached_files IS NOT NULL)::int AS has_cache,
|
|
||||||
COUNT(*) FILTER (WHERE cached_files IS NULL)::int AS no_cache
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
`);
|
|
||||||
const cf = cfRows.rows[0] as Record<string, unknown>;
|
|
||||||
report.cacheStats = cf;
|
|
||||||
|
|
||||||
console.log(` Has cached files: ${num(cf.has_cache)} (${pct(cf.has_cache, total)})`);
|
|
||||||
console.log(` No cached files: ${num(cf.no_cache)} (${pct(cf.no_cache, total)})`);
|
|
||||||
|
|
||||||
// ======================================================================
|
|
||||||
// 19. BRANCH DISTRIBUTION
|
|
||||||
// ======================================================================
|
|
||||||
header('19. BRANCH DISTRIBUTION');
|
|
||||||
|
|
||||||
const brRows = await db.execute(sql`
|
|
||||||
SELECT COALESCE(branch, 'null') AS branch, COUNT(*)::int AS count
|
|
||||||
FROM skills WHERE is_blocked = false
|
|
||||||
GROUP BY branch ORDER BY count DESC LIMIT 10
|
|
||||||
`);
|
|
||||||
report.branchDist = brRows.rows;
|
|
||||||
table(brRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
// ======================================================================
|
|
||||||
// 20. DUPLICATES
|
|
||||||
// ======================================================================
|
|
||||||
header('20. POTENTIAL DUPLICATES');
|
|
||||||
|
|
||||||
subHeader('Same Name + Description');
|
|
||||||
const ddRows = await db.execute(sql`
|
|
||||||
SELECT name, LEFT(description, 60) AS desc, COUNT(*)::int AS occurrences,
|
|
||||||
STRING_AGG(DISTINCT github_owner, ', ') AS owners
|
|
||||||
FROM skills
|
|
||||||
WHERE is_blocked = false AND description IS NOT NULL AND LENGTH(description) > 10
|
|
||||||
GROUP BY name, description HAVING COUNT(*) > 1
|
|
||||||
ORDER BY occurrences DESC LIMIT 15
|
|
||||||
`);
|
|
||||||
report.exactDupes = ddRows.rows;
|
|
||||||
table(ddRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
subHeader('Same Content Hash (identical content, 3+ copies)');
|
|
||||||
const chRows = await db.execute(sql`
|
|
||||||
SELECT content_hash, COUNT(*)::int AS copies,
|
|
||||||
MIN(name) AS sample_name,
|
|
||||||
STRING_AGG(DISTINCT github_owner, ', ') AS owners
|
|
||||||
FROM skills
|
|
||||||
WHERE is_blocked = false AND content_hash IS NOT NULL
|
|
||||||
GROUP BY content_hash HAVING COUNT(*) > 2
|
|
||||||
ORDER BY copies DESC LIMIT 15
|
|
||||||
`);
|
|
||||||
report.hashDupes = chRows.rows;
|
|
||||||
table(chRows.rows as Record<string, unknown>[]);
|
|
||||||
|
|
||||||
// ======================================================================
|
|
||||||
// DONE
|
|
||||||
// ======================================================================
|
|
||||||
const duration = ((Date.now() - startTime) / 1000).toFixed(1);
|
|
||||||
header(`EXPLORATION COMPLETE (${duration}s)`);
|
|
||||||
|
|
||||||
console.log(`
|
|
||||||
KEY QUESTIONS TO ANSWER:
|
|
||||||
1. What % of skills are SKILL.md vs other formats?
|
|
||||||
2. What % have been downloaded at least once? (real usage)
|
|
||||||
3. How concentrated is ownership? (top 10 owners = ?%)
|
|
||||||
4. How many multi-skill repos? (collection/aggregator candidates)
|
|
||||||
5. How many pass security + have good content? (curation-ready)
|
|
||||||
6. How many duplicates exist?
|
|
||||||
`);
|
|
||||||
|
|
||||||
// Save JSON report
|
|
||||||
report.generatedAt = new Date().toISOString();
|
|
||||||
report.durationSeconds = parseFloat(duration);
|
|
||||||
|
|
||||||
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
|
||||||
const reportPath = resolve(scriptDir, 'explore-report.json');
|
|
||||||
writeFileSync(reportPath, JSON.stringify(report, null, 2), 'utf-8');
|
|
||||||
console.log(` Full report saved to: ${reportPath}`);
|
|
||||||
|
|
||||||
await closeDb();
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Run ───────────────────────────────────────────────────────────────────────
|
|
||||||
runExploration().catch(async (err) => {
|
|
||||||
console.error('Exploration failed:', err);
|
|
||||||
await closeDb();
|
|
||||||
process.exit(1);
|
|
||||||
});
|
|
||||||
@@ -493,15 +493,23 @@ ALTER TABLE skills ADD COLUMN IF NOT EXISTS is_deprecated BOOLEAN DEFAULT FALSE;
|
|||||||
|
|
||||||
-- Safe pre-filter function for raw_content (handles invalid UTF-8 gracefully)
|
-- Safe pre-filter function for raw_content (handles invalid UTF-8 gracefully)
|
||||||
-- Used by review pipeline queries (getPending, countPending, countReReviews)
|
-- Used by review pipeline queries (getPending, countPending, countReReviews)
|
||||||
|
-- NOTE: EXCEPTION handler returns TRUE (benefit of the doubt) — CJK skills with
|
||||||
|
-- encoding edge cases should not be silently excluded from the review pipeline.
|
||||||
|
-- Genuine content issues will be caught during the actual review.
|
||||||
CREATE OR REPLACE FUNCTION raw_content_passes_prefilter(content TEXT) RETURNS BOOLEAN AS $$
|
CREATE OR REPLACE FUNCTION raw_content_passes_prefilter(content TEXT) RETURNS BOOLEAN AS $$
|
||||||
BEGIN
|
BEGIN
|
||||||
|
-- Length check (octet_length works on bytes, safe for any encoding)
|
||||||
IF octet_length(content) < 200 THEN RETURN FALSE; END IF;
|
IF octet_length(content) < 200 THEN RETURN FALSE; END IF;
|
||||||
IF position('<!-- generated' in content) > 0 THEN RETURN FALSE; END IF;
|
-- Content checks (may fail on encoding edge cases)
|
||||||
IF position('/Users/' in LEFT(content, 1000)) > 0 THEN RETURN FALSE; END IF;
|
BEGIN
|
||||||
IF position('C:\Users\' in LEFT(content, 1000)) > 0 THEN RETURN FALSE; END IF;
|
IF position('<!-- generated' in content) > 0 THEN RETURN FALSE; END IF;
|
||||||
|
IF position('/Users/' in LEFT(content, 1000)) > 0 THEN RETURN FALSE; END IF;
|
||||||
|
IF position('C:\Users\' in LEFT(content, 1000)) > 0 THEN RETURN FALSE; END IF;
|
||||||
|
EXCEPTION WHEN OTHERS THEN
|
||||||
|
-- Encoding error in string checks — content is long enough, let it through
|
||||||
|
RETURN TRUE;
|
||||||
|
END;
|
||||||
RETURN TRUE;
|
RETURN TRUE;
|
||||||
EXCEPTION WHEN OTHERS THEN
|
|
||||||
RETURN FALSE;
|
|
||||||
END;
|
END;
|
||||||
$$ LANGUAGE plpgsql IMMUTABLE;
|
$$ LANGUAGE plpgsql IMMUTABLE;
|
||||||
|
|
||||||
@@ -573,6 +581,10 @@ ALTER TABLE skills ADD COLUMN IF NOT EXISTS latest_ai_score INTEGER;
|
|||||||
ALTER TABLE skills ADD COLUMN IF NOT EXISTS latest_review_date TIMESTAMPTZ;
|
ALTER TABLE skills ADD COLUMN IF NOT EXISTS latest_review_date TIMESTAMPTZ;
|
||||||
CREATE INDEX IF NOT EXISTS idx_skills_ai_score ON skills(latest_ai_score) WHERE latest_ai_score IS NOT NULL;
|
CREATE INDEX IF NOT EXISTS idx_skills_ai_score ON skills(latest_ai_score) WHERE latest_ai_score IS NOT NULL;
|
||||||
|
|
||||||
|
-- Malicious skill detection (March 2026)
|
||||||
|
ALTER TABLE skills ADD COLUMN IF NOT EXISTS is_malicious BOOLEAN DEFAULT FALSE;
|
||||||
|
ALTER TABLE skill_reviews ADD COLUMN IF NOT EXISTS recommendation TEXT;
|
||||||
|
|
||||||
-- Grant permissions
|
-- Grant permissions
|
||||||
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO postgres;
|
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO postgres;
|
||||||
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO postgres;
|
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO postgres;
|
||||||
|
|||||||
Reference in New Issue
Block a user