sync: SEO canonical URLs, curation improvements, batch scripts, stats fix
- Dynamic canonical URLs and alternates for all pages (SEO fix) - Shared seo.ts utility for canonical path generation - Improved duplicate detection: standalone > aggregator priority, forks/created_at tie-breakers - browseReadyFilter now includes unique aggregator skills - Stats aligned with browseReadyFilter (was showing 16k instead of 62k+) - Re-review mechanism: content_hash changes trigger needs-re-review - review_status field added to skills schema - Batch security scan and quality score scripts - Indexer Dockerfile updated for curation deps - Removed roadmap from README Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
airano
240
scripts/curation/batch-security.mjs
Normal file
240
scripts/curation/batch-security.mjs
Normal file
@@ -0,0 +1,240 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Phase 4: Batch Security Scan
|
||||
*
|
||||
* Scans browse-ready skills for security issues using scanSecurity() from skillhub-core.
|
||||
* Updates: security_score, security_status, last_scanned, review_status → 'auto-scored'
|
||||
*
|
||||
* Usage:
|
||||
* DATABASE_URL=postgres://... node scripts/curation/batch-security.mjs
|
||||
*
|
||||
* Options:
|
||||
* --dry-run Show what would change without writing
|
||||
* --batch-size=N Process N skills per batch (default 100)
|
||||
*/
|
||||
|
||||
import { createRequire } from 'module';
|
||||
import { resolve, dirname } from 'path';
|
||||
import { fileURLToPath, pathToFileURL } from 'url';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
// ─── Find pg module ───
|
||||
let pg;
|
||||
const tryPaths = [
|
||||
resolve(__dirname, '../..', 'package.json'),
|
||||
'/tmp/package.json',
|
||||
process.env.APPDATA ? resolve(process.env.APPDATA, '..', 'Local', 'Temp', 'package.json') : null,
|
||||
].filter(Boolean);
|
||||
for (const p of tryPaths) {
|
||||
try { pg = createRequire(p)('pg'); break; } catch {}
|
||||
}
|
||||
if (!pg) { console.error('pg not found. Run: npm install pg'); process.exit(1); }
|
||||
|
||||
// ─── Find skillhub-core (ESM-only — use direct path import) ───
|
||||
let scanSecurity;
|
||||
const corePaths = [
|
||||
resolve(__dirname, '../../packages/core/dist/index.js'),
|
||||
resolve(__dirname, '../../node_modules/skillhub-core/dist/index.js'),
|
||||
resolve(__dirname, '../../services/indexer/node_modules/skillhub-core/dist/index.js'),
|
||||
];
|
||||
for (const p of corePaths) {
|
||||
try {
|
||||
const core = await import(pathToFileURL(p).href);
|
||||
scanSecurity = core.scanSecurity;
|
||||
break;
|
||||
} catch {}
|
||||
}
|
||||
if (!scanSecurity) { console.error('skillhub-core not found. Run: pnpm build'); process.exit(1); }
|
||||
|
||||
// ─── Config ───
|
||||
const DATABASE_URL = process.env.DATABASE_URL || 'postgresql://postgres:postgres@localhost:5432/skillhub';
|
||||
const DRY_RUN = process.argv.includes('--dry-run');
|
||||
const batchArg = process.argv.find(a => a.startsWith('--batch-size='));
|
||||
const BATCH_SIZE = batchArg ? parseInt(batchArg.split('=')[1]) : 100;
|
||||
|
||||
// ─── Helpers ───
|
||||
const n = v => Number(v ?? 0).toLocaleString('en-US');
|
||||
|
||||
function progress(current, total) {
|
||||
const pct = ((current / total) * 100).toFixed(1);
|
||||
process.stdout.write(`\r Processing ${n(current)} / ${n(total)} (${pct}%)`);
|
||||
}
|
||||
|
||||
// ─── Database ───
|
||||
let client;
|
||||
|
||||
async function connect() {
|
||||
client = new pg.Client({
|
||||
connectionString: DATABASE_URL,
|
||||
ssl: false,
|
||||
connectionTimeoutMillis: 15000,
|
||||
query_timeout: 600000,
|
||||
keepAlive: true,
|
||||
});
|
||||
await client.connect();
|
||||
console.log('Connected to database');
|
||||
}
|
||||
|
||||
async function query(sql, params = []) {
|
||||
return client.query(sql, params);
|
||||
}
|
||||
|
||||
// ─── Extract scripts from cached_files ───
|
||||
function extractScripts(cachedFiles) {
|
||||
if (!cachedFiles || !cachedFiles.items) return [];
|
||||
return cachedFiles.items
|
||||
.filter(item =>
|
||||
!item.isBinary &&
|
||||
item.name !== 'SKILL.md' &&
|
||||
(item.name.endsWith('.sh') ||
|
||||
item.name.endsWith('.py') ||
|
||||
item.name.endsWith('.js') ||
|
||||
item.name.endsWith('.ts') ||
|
||||
item.name.endsWith('.ps1') ||
|
||||
item.name.endsWith('.bat') ||
|
||||
item.name.endsWith('.rb'))
|
||||
)
|
||||
.map(item => ({ name: item.name, content: item.content }));
|
||||
}
|
||||
|
||||
// ─── Main ───
|
||||
async function main() {
|
||||
const t0 = Date.now();
|
||||
await connect();
|
||||
|
||||
console.log(`\n${'='.repeat(70)}`);
|
||||
console.log(' BATCH SECURITY SCAN (Phase 4)');
|
||||
console.log(`${'='.repeat(70)}`);
|
||||
if (DRY_RUN) console.log('\n *** DRY RUN MODE — no changes will be made ***\n');
|
||||
|
||||
// Count skills to scan
|
||||
const countResult = await query(`
|
||||
SELECT COUNT(*)::int AS total
|
||||
FROM skills
|
||||
WHERE is_blocked = false
|
||||
AND is_duplicate = false
|
||||
AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection'))
|
||||
AND security_status IS NULL
|
||||
AND raw_content IS NOT NULL
|
||||
`);
|
||||
const total = countResult.rows[0].total;
|
||||
console.log(` Skills to scan: ${n(total)}`);
|
||||
console.log(` Batch size: ${BATCH_SIZE}`);
|
||||
|
||||
if (total === 0) {
|
||||
console.log(' Nothing to do — all browse-ready skills already scanned');
|
||||
await client.end().catch(() => {});
|
||||
return;
|
||||
}
|
||||
|
||||
if (DRY_RUN) {
|
||||
// Show sample
|
||||
const sample = await query(`
|
||||
SELECT id, LEFT(name, 50) AS name
|
||||
FROM skills
|
||||
WHERE is_blocked = false
|
||||
AND is_duplicate = false
|
||||
AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection'))
|
||||
AND security_status IS NULL
|
||||
AND raw_content IS NOT NULL
|
||||
LIMIT 5
|
||||
`);
|
||||
console.log('\n Sample skills that would be scanned:');
|
||||
for (const r of sample.rows) {
|
||||
console.log(` ${r.id} — ${r.name}`);
|
||||
}
|
||||
console.log(`\n [DRY RUN] Would scan ${n(total)} skills`);
|
||||
await client.end().catch(() => {});
|
||||
return;
|
||||
}
|
||||
|
||||
// Process in batches
|
||||
let processed = 0;
|
||||
let passCount = 0;
|
||||
let warnCount = 0;
|
||||
let failCount = 0;
|
||||
let errorCount = 0;
|
||||
let totalScore = 0;
|
||||
|
||||
while (processed < total) {
|
||||
const batch = await query(`
|
||||
SELECT id, raw_content, cached_files
|
||||
FROM skills
|
||||
WHERE is_blocked = false
|
||||
AND is_duplicate = false
|
||||
AND (skill_type IS NULL OR skill_type IN ('standalone', 'collection'))
|
||||
AND security_status IS NULL
|
||||
AND raw_content IS NOT NULL
|
||||
ORDER BY github_stars DESC NULLS LAST
|
||||
LIMIT $1
|
||||
`, [BATCH_SIZE]);
|
||||
|
||||
if (batch.rows.length === 0) break;
|
||||
|
||||
for (const row of batch.rows) {
|
||||
try {
|
||||
const scripts = extractScripts(row.cached_files);
|
||||
const report = scanSecurity({ content: row.raw_content, scripts });
|
||||
|
||||
await query(`
|
||||
UPDATE skills
|
||||
SET security_score = $1,
|
||||
security_status = $2,
|
||||
last_scanned = NOW(),
|
||||
review_status = CASE
|
||||
WHEN review_status IS NULL OR review_status = 'unreviewed'
|
||||
THEN 'auto-scored'
|
||||
ELSE review_status
|
||||
END
|
||||
WHERE id = $3
|
||||
`, [report.score, report.status, row.id]);
|
||||
|
||||
totalScore += report.score;
|
||||
if (report.status === 'pass') passCount++;
|
||||
else if (report.status === 'warning') warnCount++;
|
||||
else failCount++;
|
||||
} catch (err) {
|
||||
errorCount++;
|
||||
if (errorCount <= 5) {
|
||||
console.error(`\n Error scanning ${row.id}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
processed++;
|
||||
if (processed % 100 === 0 || processed === total) {
|
||||
progress(processed, total);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\n'); // Clear progress line
|
||||
|
||||
// Summary
|
||||
const avgScore = processed > 0 ? (totalScore / (processed - errorCount)).toFixed(1) : 0;
|
||||
console.log(`
|
||||
┌─────────────────────────────────────────────────────┐
|
||||
│ SECURITY SCAN SUMMARY │
|
||||
├─────────────────────────────────────────────────────┤
|
||||
│ Total scanned: ${n(processed).padStart(8)} │
|
||||
│ Errors (skipped): ${n(errorCount).padStart(8)} │
|
||||
├─────────────────────────────────────────────────────┤
|
||||
│ PASS: ${n(passCount).padStart(8)} │
|
||||
│ WARNING: ${n(warnCount).padStart(8)} │
|
||||
│ FAIL: ${n(failCount).padStart(8)} │
|
||||
├─────────────────────────────────────────────────────┤
|
||||
│ Avg security score:${avgScore.toString().padStart(8)} │
|
||||
└─────────────────────────────────────────────────────┘
|
||||
`);
|
||||
|
||||
const dur = ((Date.now() - t0) / 1000).toFixed(1);
|
||||
console.log(`Completed in ${dur}s`);
|
||||
|
||||
await client.end().catch(() => {});
|
||||
}
|
||||
|
||||
main().catch(async e => {
|
||||
console.error('FAILED:', e.message || e);
|
||||
await client?.end().catch(() => {});
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user