Initial release v1.0.0
Open-source marketplace for AI Agent skills. Features: - Next.js 15 web app with i18n (en/fa) - CLI tool for skill installation (npx skillhub) - GitHub crawler/indexer with multi-strategy discovery - Security scanning for all indexed skills - Self-hostable with Docker Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
airano
5
services/indexer/.eslintrc.json
Normal file
5
services/indexer/.eslintrc.json
Normal file
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"rules": {
|
||||
"no-console": "off"
|
||||
}
|
||||
}
|
||||
49
services/indexer/Dockerfile
Normal file
49
services/indexer/Dockerfile
Normal file
@@ -0,0 +1,49 @@
|
||||
# Stage 1: Dependencies
|
||||
FROM node:20-alpine AS deps
|
||||
RUN npm install -g pnpm@9
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy workspace configuration
|
||||
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml turbo.json ./
|
||||
COPY services/indexer/package.json ./services/indexer/
|
||||
COPY packages/core/package.json ./packages/core/
|
||||
COPY packages/db/package.json ./packages/db/
|
||||
|
||||
# Install dependencies
|
||||
RUN pnpm install --frozen-lockfile
|
||||
|
||||
# Stage 2: Builder
|
||||
FROM node:20-alpine AS builder
|
||||
RUN npm install -g pnpm@9
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy dependencies
|
||||
COPY --from=deps /app/node_modules ./node_modules
|
||||
COPY --from=deps /app/services/indexer/node_modules ./services/indexer/node_modules
|
||||
COPY --from=deps /app/packages ./packages
|
||||
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Build the application
|
||||
RUN pnpm turbo run build --filter=@skillhub/indexer
|
||||
|
||||
# Stage 3: Runner (minimal - all deps are bundled)
|
||||
FROM node:20-alpine AS runner
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
ENV NODE_ENV=production
|
||||
|
||||
# Create non-root user
|
||||
RUN addgroup --system --gid 1001 nodejs
|
||||
RUN adduser --system --uid 1001 indexer
|
||||
|
||||
# Copy only the bundled dist (no node_modules needed - everything is bundled)
|
||||
COPY --from=builder --chown=indexer:nodejs /app/services/indexer/dist ./dist
|
||||
|
||||
USER indexer
|
||||
|
||||
CMD ["node", "dist/worker.js"]
|
||||
31
services/indexer/package.json
Normal file
31
services/indexer/package.json
Normal file
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "@skillhub/indexer",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"build": "tsup",
|
||||
"dev": "tsx watch src/index.ts",
|
||||
"start": "node dist/index.js",
|
||||
"worker": "tsx src/worker.ts",
|
||||
"crawl": "tsx src/crawl.ts",
|
||||
"lint": "eslint src/",
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@octokit/rest": "^20.0.2",
|
||||
"skillhub-core": "workspace:*",
|
||||
"@skillhub/db": "workspace:*",
|
||||
"bullmq": "^5.1.0",
|
||||
"ioredis": "^5.3.2",
|
||||
"meilisearch": "^0.55.0",
|
||||
"p-limit": "^5.0.0",
|
||||
"resend": "^6.9.1",
|
||||
"zod": "^3.22.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.10.0",
|
||||
"tsup": "^8.0.1",
|
||||
"tsx": "^4.7.0",
|
||||
"typescript": "^5.3.0"
|
||||
}
|
||||
}
|
||||
365
services/indexer/src/analyzer.ts
Normal file
365
services/indexer/src/analyzer.ts
Normal file
@@ -0,0 +1,365 @@
|
||||
import {
|
||||
parseSkillMd,
|
||||
parseGenericInstructionFile,
|
||||
validateSkill,
|
||||
scanSecurity,
|
||||
type ParsedSkill,
|
||||
type SecurityReport,
|
||||
type ValidationResult,
|
||||
type SourceFormat,
|
||||
} from 'skillhub-core';
|
||||
import type { SkillContent, RepoMetadata } from './crawler.js';
|
||||
|
||||
export interface AnalysisResult {
|
||||
skill: ParsedSkill;
|
||||
security: SecurityReport;
|
||||
validation: ValidationResult;
|
||||
quality: QualityScore;
|
||||
meta: AnalysisMeta;
|
||||
}
|
||||
|
||||
export interface QualityScore {
|
||||
overall: number; // 0-100
|
||||
documentation: number;
|
||||
maintenance: number;
|
||||
popularity: number;
|
||||
factors: QualityFactor[];
|
||||
}
|
||||
|
||||
export interface QualityFactor {
|
||||
name: string;
|
||||
score: number;
|
||||
weight: number;
|
||||
details?: string;
|
||||
}
|
||||
|
||||
export interface AnalysisMeta {
|
||||
analyzedAt: Date;
|
||||
contentHash: string;
|
||||
version?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze a skill for quality, security, and validity
|
||||
*/
|
||||
export class SkillAnalyzer {
|
||||
/**
|
||||
* Perform full analysis of a skill
|
||||
*/
|
||||
analyze(content: SkillContent, sourceFormat: SourceFormat = 'skill.md'): AnalysisResult {
|
||||
let skill: ParsedSkill;
|
||||
|
||||
if (sourceFormat === 'skill.md') {
|
||||
skill = parseSkillMd(content.skillMd);
|
||||
} else {
|
||||
skill = parseGenericInstructionFile(content.skillMd, sourceFormat, {
|
||||
name: content.repoMeta.description?.split(/\s+/).slice(0, 3).join('-') || 'skill',
|
||||
description: content.repoMeta.description,
|
||||
owner: '',
|
||||
});
|
||||
}
|
||||
|
||||
// Run validation: relaxed for non-SKILL.md formats
|
||||
const validation = sourceFormat === 'skill.md'
|
||||
? validateSkill(skill)
|
||||
: this.validateGenericFile(skill);
|
||||
|
||||
// Run security scan (works the same for all formats)
|
||||
const security = scanSecurity({
|
||||
content: content.skillMd,
|
||||
scripts: content.scripts.map((s) => ({
|
||||
name: s.name,
|
||||
content: s.content,
|
||||
})),
|
||||
});
|
||||
|
||||
// Calculate quality score
|
||||
const quality = this.calculateQuality(skill, content, security, validation);
|
||||
|
||||
// Generate content hash
|
||||
const contentHash = this.hashContent(content.skillMd);
|
||||
|
||||
return {
|
||||
skill,
|
||||
security,
|
||||
validation,
|
||||
quality,
|
||||
meta: {
|
||||
analyzedAt: new Date(),
|
||||
contentHash,
|
||||
version: skill.metadata.version,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Relaxed validation for non-SKILL.md formats
|
||||
*/
|
||||
private validateGenericFile(skill: ParsedSkill): ValidationResult {
|
||||
const errors = [...skill.validation.errors];
|
||||
const warnings = [...skill.validation.warnings];
|
||||
|
||||
if (!skill.content || skill.content.trim().length === 0) {
|
||||
errors.push({
|
||||
code: 'EMPTY_CONTENT',
|
||||
message: 'Instruction file content is empty',
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
isValid: errors.length === 0,
|
||||
errors,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate quality score based on multiple factors
|
||||
*/
|
||||
private calculateQuality(
|
||||
skill: ParsedSkill,
|
||||
content: SkillContent,
|
||||
security: SecurityReport,
|
||||
validation: ValidationResult
|
||||
): QualityScore {
|
||||
const factors: QualityFactor[] = [];
|
||||
|
||||
// Documentation quality (30% weight)
|
||||
const docScore = this.scoreDocumentation(skill, content);
|
||||
factors.push({
|
||||
name: 'documentation',
|
||||
score: docScore,
|
||||
weight: 0.3,
|
||||
details: this.getDocDetails(skill),
|
||||
});
|
||||
|
||||
// Maintenance signals (25% weight)
|
||||
const maintScore = this.scoreMaintenance(content.repoMeta);
|
||||
factors.push({
|
||||
name: 'maintenance',
|
||||
score: maintScore,
|
||||
weight: 0.25,
|
||||
details: this.getMaintDetails(content.repoMeta),
|
||||
});
|
||||
|
||||
// Popularity (20% weight)
|
||||
const popScore = this.scorePopularity(content.repoMeta);
|
||||
factors.push({
|
||||
name: 'popularity',
|
||||
score: popScore,
|
||||
weight: 0.2,
|
||||
});
|
||||
|
||||
// Security (15% weight)
|
||||
factors.push({
|
||||
name: 'security',
|
||||
score: security.score,
|
||||
weight: 0.15,
|
||||
});
|
||||
|
||||
// Validation (10% weight)
|
||||
const valScore = validation.isValid ? 100 : Math.max(0, 100 - validation.errors.length * 20);
|
||||
factors.push({
|
||||
name: 'validation',
|
||||
score: valScore,
|
||||
weight: 0.1,
|
||||
});
|
||||
|
||||
// Calculate weighted overall score
|
||||
const overall = Math.round(
|
||||
factors.reduce((sum, f) => sum + f.score * f.weight, 0)
|
||||
);
|
||||
|
||||
return {
|
||||
overall,
|
||||
documentation: docScore,
|
||||
maintenance: maintScore,
|
||||
popularity: popScore,
|
||||
factors,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Score documentation quality
|
||||
*/
|
||||
private scoreDocumentation(skill: ParsedSkill, content: SkillContent): number {
|
||||
let score = 0;
|
||||
|
||||
// Has description (required)
|
||||
if (skill.metadata.description && skill.metadata.description.length > 20) {
|
||||
score += 20;
|
||||
}
|
||||
|
||||
// Content length and structure
|
||||
const contentLength = skill.content.length;
|
||||
if (contentLength > 500) score += 15;
|
||||
else if (contentLength > 200) score += 10;
|
||||
else if (contentLength > 50) score += 5;
|
||||
|
||||
// Has headers (good structure)
|
||||
const headerCount = (skill.content.match(/^#+\s/gm) || []).length;
|
||||
if (headerCount >= 3) score += 15;
|
||||
else if (headerCount >= 1) score += 10;
|
||||
|
||||
// Has code examples
|
||||
if (skill.content.includes('```')) {
|
||||
score += 15;
|
||||
}
|
||||
|
||||
// Has version
|
||||
if (skill.metadata.version) {
|
||||
score += 10;
|
||||
}
|
||||
|
||||
// Has license
|
||||
if (skill.metadata.license) {
|
||||
score += 5;
|
||||
}
|
||||
|
||||
// Has compatibility info
|
||||
if (skill.metadata.compatibility?.platforms?.length) {
|
||||
score += 10;
|
||||
}
|
||||
|
||||
// Has scripts
|
||||
if (content.scripts.length > 0) {
|
||||
score += 5;
|
||||
}
|
||||
|
||||
// Has references
|
||||
if (content.references.length > 0) {
|
||||
score += 5;
|
||||
}
|
||||
|
||||
return Math.min(100, score);
|
||||
}
|
||||
|
||||
/**
|
||||
* Score maintenance based on repo activity
|
||||
*/
|
||||
private scoreMaintenance(repoMeta: RepoMetadata): number {
|
||||
let score = 0;
|
||||
|
||||
// Check last update time
|
||||
const lastUpdate = new Date(repoMeta.updatedAt);
|
||||
const daysSinceUpdate = (Date.now() - lastUpdate.getTime()) / (1000 * 60 * 60 * 24);
|
||||
|
||||
if (daysSinceUpdate < 30) score += 40;
|
||||
else if (daysSinceUpdate < 90) score += 30;
|
||||
else if (daysSinceUpdate < 180) score += 20;
|
||||
else if (daysSinceUpdate < 365) score += 10;
|
||||
|
||||
// Has license
|
||||
if (repoMeta.license) {
|
||||
score += 20;
|
||||
}
|
||||
|
||||
// Has description
|
||||
if (repoMeta.description) {
|
||||
score += 10;
|
||||
}
|
||||
|
||||
// Has topics
|
||||
if (repoMeta.topics.length > 0) {
|
||||
score += 10;
|
||||
}
|
||||
|
||||
// Activity level (forks indicate usage)
|
||||
if (repoMeta.forks >= 10) score += 20;
|
||||
else if (repoMeta.forks >= 5) score += 15;
|
||||
else if (repoMeta.forks >= 1) score += 10;
|
||||
|
||||
return Math.min(100, score);
|
||||
}
|
||||
|
||||
/**
|
||||
* Score popularity based on stars and forks
|
||||
*/
|
||||
private scorePopularity(repoMeta: RepoMetadata): number {
|
||||
const stars = repoMeta.stars;
|
||||
const forks = repoMeta.forks;
|
||||
|
||||
// Logarithmic scale for stars (more stars = diminishing returns)
|
||||
let score = 0;
|
||||
|
||||
if (stars >= 1000) score += 50;
|
||||
else if (stars >= 100) score += 40;
|
||||
else if (stars >= 50) score += 30;
|
||||
else if (stars >= 10) score += 20;
|
||||
else if (stars >= 5) score += 10;
|
||||
else if (stars >= 1) score += 5;
|
||||
|
||||
// Bonus for forks
|
||||
if (forks >= 50) score += 30;
|
||||
else if (forks >= 10) score += 20;
|
||||
else if (forks >= 5) score += 15;
|
||||
else if (forks >= 1) score += 10;
|
||||
|
||||
// Bonus for relevant topics
|
||||
const relevantTopics = ['ai', 'agent', 'skill', 'claude', 'copilot', 'codex', 'llm'];
|
||||
const hasRelevantTopic = repoMeta.topics.some((t) =>
|
||||
relevantTopics.some((rt) => t.toLowerCase().includes(rt))
|
||||
);
|
||||
if (hasRelevantTopic) {
|
||||
score += 20;
|
||||
}
|
||||
|
||||
return Math.min(100, score);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get documentation details for display
|
||||
*/
|
||||
private getDocDetails(skill: ParsedSkill): string {
|
||||
const parts: string[] = [];
|
||||
|
||||
if (skill.metadata.version) {
|
||||
parts.push(`v${skill.metadata.version}`);
|
||||
}
|
||||
|
||||
if (skill.metadata.license) {
|
||||
parts.push(skill.metadata.license);
|
||||
}
|
||||
|
||||
const platforms = skill.metadata.compatibility?.platforms;
|
||||
if (platforms?.length) {
|
||||
parts.push(platforms.join(', '));
|
||||
}
|
||||
|
||||
return parts.join(' | ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get maintenance details for display
|
||||
*/
|
||||
private getMaintDetails(repoMeta: RepoMetadata): string {
|
||||
const lastUpdate = new Date(repoMeta.updatedAt);
|
||||
const daysAgo = Math.floor((Date.now() - lastUpdate.getTime()) / (1000 * 60 * 60 * 24));
|
||||
|
||||
if (daysAgo === 0) return 'Updated today';
|
||||
if (daysAgo === 1) return 'Updated yesterday';
|
||||
if (daysAgo < 7) return `Updated ${daysAgo} days ago`;
|
||||
if (daysAgo < 30) return `Updated ${Math.floor(daysAgo / 7)} weeks ago`;
|
||||
if (daysAgo < 365) return `Updated ${Math.floor(daysAgo / 30)} months ago`;
|
||||
return `Updated ${Math.floor(daysAgo / 365)} years ago`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a simple hash of content for change detection
|
||||
*/
|
||||
private hashContent(content: string): string {
|
||||
let hash = 0;
|
||||
for (let i = 0; i < content.length; i++) {
|
||||
const char = content.charCodeAt(i);
|
||||
hash = ((hash << 5) - hash + char) | 0;
|
||||
}
|
||||
return Math.abs(hash).toString(16).padStart(8, '0');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new SkillAnalyzer instance
|
||||
*/
|
||||
export function createAnalyzer(): SkillAnalyzer {
|
||||
return new SkillAnalyzer();
|
||||
}
|
||||
779
services/indexer/src/crawl.ts
Normal file
779
services/indexer/src/crawl.ts
Normal file
@@ -0,0 +1,779 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* CLI script to manually trigger a crawl job or sync operations
|
||||
*/
|
||||
|
||||
import { INSTRUCTION_FILE_PATTERNS, type SourceFormat } from 'skillhub-core';
|
||||
import { scheduleFullCrawl, scheduleIncrementalCrawl, getQueueStats, getQueue } from './queue.js';
|
||||
import { syncAllSkillsToMeilisearch, checkMeilisearchHealth } from './meilisearch-sync.js';
|
||||
import { createDb, skillQueries, categoryQueries, discoveredRepoQueries, awesomeListQueries, addRequestQueries, userQueries } from '@skillhub/db';
|
||||
import { createStrategyOrchestrator, createDeepScanCrawler, createAwesomeListCrawler } from './strategies/index.js';
|
||||
import { createCrawler } from './crawler.js';
|
||||
import { indexSkill } from './skill-indexer.js';
|
||||
import { TokenManager } from './token-manager.js';
|
||||
|
||||
const command = process.argv[2] || 'stats';
|
||||
|
||||
async function main() {
|
||||
switch (command) {
|
||||
case 'full': {
|
||||
console.log('Scheduling full crawl...');
|
||||
const fullJobId = await scheduleFullCrawl({
|
||||
minStars: parseInt(process.env.INDEXER_MIN_STARS || process.env.MIN_STARS || '0'),
|
||||
});
|
||||
console.log(`Full crawl scheduled with job ID: ${fullJobId}`);
|
||||
break;
|
||||
}
|
||||
|
||||
case 'incremental': {
|
||||
console.log('Scheduling incremental crawl...');
|
||||
const incJobId = await scheduleIncrementalCrawl();
|
||||
console.log(`Incremental crawl scheduled with job ID: ${incJobId}`);
|
||||
break;
|
||||
}
|
||||
|
||||
case 'sync-meili': {
|
||||
console.log('Syncing all skills to Meilisearch (streaming mode)...\n');
|
||||
|
||||
// Check Meilisearch connection
|
||||
const healthy = await checkMeilisearchHealth();
|
||||
if (!healthy) {
|
||||
console.error('Meilisearch is not reachable. Please check MEILI_URL and MEILI_MASTER_KEY');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Stream skills from database to Meilisearch in batches
|
||||
// This approach is memory-efficient and scales to any number of skills
|
||||
const db = createDb(process.env.DATABASE_URL);
|
||||
const SYNC_BATCH_SIZE = 5000;
|
||||
let syncOffset = 0;
|
||||
let totalSuccess = 0;
|
||||
let totalFailed = 0;
|
||||
let batchNumber = 0;
|
||||
|
||||
// First, get total count for progress display
|
||||
const totalCount = await skillQueries.count(db, {});
|
||||
console.log(`Total skills in database: ${totalCount}`);
|
||||
|
||||
if (totalCount === 0) {
|
||||
console.log('No skills found in database');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const totalBatches = Math.ceil(totalCount / SYNC_BATCH_SIZE);
|
||||
console.log(`Will process in ${totalBatches} batches of ${SYNC_BATCH_SIZE}\n`);
|
||||
|
||||
for (;;) {
|
||||
// Fetch batch from database
|
||||
const batch = await skillQueries.search(db, { limit: SYNC_BATCH_SIZE, offset: syncOffset });
|
||||
if (batch.length === 0) break;
|
||||
|
||||
batchNumber++;
|
||||
|
||||
// Immediately sync this batch to Meilisearch (no memory accumulation)
|
||||
const results = await syncAllSkillsToMeilisearch(batch);
|
||||
totalSuccess += results.success;
|
||||
totalFailed += results.failed;
|
||||
|
||||
const progress = Math.min(100, Math.round((syncOffset + batch.length) / totalCount * 100));
|
||||
console.log(`Batch ${batchNumber}/${totalBatches}: ${results.success} synced, ${results.failed} failed (${progress}% complete)`);
|
||||
|
||||
if (batch.length < SYNC_BATCH_SIZE) break;
|
||||
syncOffset += SYNC_BATCH_SIZE;
|
||||
}
|
||||
|
||||
console.log(`\n════════════════════════════════════════`);
|
||||
console.log(`Sync complete:`);
|
||||
console.log(` Total processed: ${totalSuccess + totalFailed}`);
|
||||
console.log(` Success: ${totalSuccess}`);
|
||||
console.log(` Failed: ${totalFailed}`);
|
||||
break;
|
||||
}
|
||||
|
||||
case 'stats': {
|
||||
const q = getQueue();
|
||||
console.log('Queue statistics:');
|
||||
const stats = await getQueueStats();
|
||||
console.log(` Waiting: ${stats.waiting}`);
|
||||
console.log(` Active: ${stats.active}`);
|
||||
console.log(` Completed: ${stats.completed}`);
|
||||
console.log(` Failed: ${stats.failed}`);
|
||||
console.log(` Delayed: ${stats.delayed}`);
|
||||
|
||||
// Show failed job details if any
|
||||
if (stats.failed > 0) {
|
||||
const failedJobs = await q.getFailed(0, 10);
|
||||
if (failedJobs.length > 0) {
|
||||
console.log('\nFailed jobs:');
|
||||
for (const job of failedJobs) {
|
||||
const failedAt = job.finishedOn ? new Date(job.finishedOn).toISOString() : 'unknown';
|
||||
console.log(` - [${job.data.type}] ${job.name} (${failedAt})`);
|
||||
if (job.failedReason) {
|
||||
console.log(` Reason: ${job.failedReason.slice(0, 120)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case 'token-status': {
|
||||
console.log('\n═══ GitHub Token Status ═══');
|
||||
const tokenManager = TokenManager.getInstance();
|
||||
|
||||
// Refresh all tokens from GitHub API first
|
||||
console.log('Fetching latest status from GitHub API...\n');
|
||||
const tokenStatus = tokenManager.getStatus();
|
||||
for (const tokenInfo of tokenStatus.tokens) {
|
||||
await tokenManager.refreshRateLimit(tokenInfo.token);
|
||||
}
|
||||
|
||||
// Get updated status
|
||||
const updatedStatus = tokenManager.getStatus();
|
||||
|
||||
console.log(`\nTotal Tokens: ${updatedStatus.totalTokens}`);
|
||||
console.log(`Available Tokens: ${updatedStatus.availableTokens}`);
|
||||
console.log(`Global Remaining: ${updatedStatus.globalRemaining}`);
|
||||
console.log(`Next Reset: ${new Date(updatedStatus.nextReset).toLocaleTimeString()}`);
|
||||
console.log('\nToken Details:');
|
||||
|
||||
for (const token of updatedStatus.tokens) {
|
||||
const status = token.isExhausted ? '❌ EXHAUSTED' : '✅ AVAILABLE';
|
||||
const resetTime = new Date(token.reset).toLocaleTimeString();
|
||||
console.log(` [${token.name}] ${status}`);
|
||||
console.log(` Remaining: ${token.remaining}/${token.limit}`);
|
||||
console.log(` Resets at: ${resetTime}`);
|
||||
console.log(` Last used: ${token.lastUsed ? new Date(token.lastUsed).toLocaleString() : 'Never'}`);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case 'link-categories':
|
||||
case 'recategorize': {
|
||||
console.log('Re-categorizing all skills with 23 categories (streaming mode)...\n');
|
||||
const dbCat = createDb(process.env.DATABASE_URL);
|
||||
|
||||
// First, get total count for progress display
|
||||
const linkTotalCount = await skillQueries.count(dbCat, {});
|
||||
console.log(`Total skills in database: ${linkTotalCount}`);
|
||||
|
||||
if (linkTotalCount === 0) {
|
||||
console.log('No skills found in database');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Stream skills from database and link categories in batches
|
||||
const LINK_BATCH_SIZE = 1000;
|
||||
let linkOffset = 0;
|
||||
let linked = 0;
|
||||
let failed = 0;
|
||||
let batchNum = 0;
|
||||
const totalBatches = Math.ceil(linkTotalCount / LINK_BATCH_SIZE);
|
||||
|
||||
console.log(`Will process in ${totalBatches} batches of ${LINK_BATCH_SIZE}\n`);
|
||||
|
||||
for (;;) {
|
||||
const batch = await skillQueries.search(dbCat, { limit: LINK_BATCH_SIZE, offset: linkOffset });
|
||||
if (batch.length === 0) break;
|
||||
|
||||
batchNum++;
|
||||
|
||||
// Process this batch immediately (no memory accumulation)
|
||||
let batchLinked = 0;
|
||||
let batchFailed = 0;
|
||||
|
||||
for (const skill of batch) {
|
||||
try {
|
||||
await categoryQueries.linkSkillToCategories(
|
||||
dbCat,
|
||||
skill.id,
|
||||
skill.name,
|
||||
skill.description || ''
|
||||
);
|
||||
batchLinked++;
|
||||
} catch {
|
||||
batchFailed++;
|
||||
}
|
||||
}
|
||||
|
||||
linked += batchLinked;
|
||||
failed += batchFailed;
|
||||
|
||||
const progress = Math.min(100, Math.round((linkOffset + batch.length) / linkTotalCount * 100));
|
||||
console.log(`Batch ${batchNum}/${totalBatches}: ${batchLinked} linked, ${batchFailed} failed (${progress}% complete)`);
|
||||
|
||||
if (batch.length < LINK_BATCH_SIZE) break;
|
||||
linkOffset += LINK_BATCH_SIZE;
|
||||
}
|
||||
|
||||
// No need to manually update category counts - database trigger handles this automatically
|
||||
// (See init-db.sql lines 356-373: update_category_count trigger)
|
||||
|
||||
console.log(`\n════════════════════════════════════════`);
|
||||
console.log(`Linking complete:`);
|
||||
console.log(` Total processed: ${linked + failed}`);
|
||||
console.log(` Linked: ${linked}`);
|
||||
console.log(` Failed: ${failed}`);
|
||||
break;
|
||||
}
|
||||
|
||||
case 'discover-repos': {
|
||||
console.log('Running all discovery strategies...\n');
|
||||
const discoverDb = createDb(process.env.DATABASE_URL);
|
||||
const orchestrator = createStrategyOrchestrator();
|
||||
|
||||
const { repos: discoveredRepos, stats: discoverStats } = await orchestrator.runAllStrategies();
|
||||
|
||||
console.log(`\nSaving ${discoveredRepos.length} discovered repos to database...`);
|
||||
let savedRepos = 0;
|
||||
for (const repo of discoveredRepos) {
|
||||
try {
|
||||
await discoveredRepoQueries.upsert(discoverDb, {
|
||||
id: `${repo.owner}/${repo.repo}`,
|
||||
owner: repo.owner,
|
||||
repo: repo.repo,
|
||||
discoveredVia: repo.discoveredVia,
|
||||
githubStars: repo.stars,
|
||||
});
|
||||
savedRepos++;
|
||||
} catch {
|
||||
// Skip duplicates
|
||||
}
|
||||
}
|
||||
console.log(`Saved ${savedRepos} new repositories`);
|
||||
console.log(`\nDiscovery complete in ${(discoverStats.duration / 1000).toFixed(1)}s`);
|
||||
break;
|
||||
}
|
||||
|
||||
case 'awesome-lists': {
|
||||
console.log('Running awesome list discovery...\n');
|
||||
const awesomeDb = createDb(process.env.DATABASE_URL);
|
||||
const awesomeCrawler = createAwesomeListCrawler();
|
||||
|
||||
// Save known lists to DB
|
||||
for (const list of awesomeCrawler.getKnownLists()) {
|
||||
await awesomeListQueries.upsert(awesomeDb, {
|
||||
id: `${list.owner}/${list.repo}`,
|
||||
owner: list.owner,
|
||||
repo: list.repo,
|
||||
name: `${list.owner}/${list.repo}`,
|
||||
});
|
||||
}
|
||||
|
||||
const listResults = await awesomeCrawler.crawlAllLists();
|
||||
let awesomeTotal = 0;
|
||||
|
||||
for (const [listId, repoRefs] of listResults.entries()) {
|
||||
console.log(`\n${listId}: ${repoRefs.length} repos`);
|
||||
|
||||
// Update list stats
|
||||
await awesomeListQueries.markParsed(awesomeDb, listId, repoRefs.length);
|
||||
|
||||
// Save repos
|
||||
for (const ref of repoRefs) {
|
||||
try {
|
||||
await discoveredRepoQueries.upsert(awesomeDb, {
|
||||
id: `${ref.owner}/${ref.repo}`,
|
||||
owner: ref.owner,
|
||||
repo: ref.repo,
|
||||
discoveredVia: 'awesome-list',
|
||||
sourceUrl: `https://github.com/${listId}`,
|
||||
});
|
||||
awesomeTotal++;
|
||||
} catch {
|
||||
// Skip duplicates
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\nSaved ${awesomeTotal} repositories from awesome lists`);
|
||||
break;
|
||||
}
|
||||
|
||||
case 'deep-scan': {
|
||||
console.log('Deep scanning discovered repositories...\n');
|
||||
const deepDb = createDb(process.env.DATABASE_URL);
|
||||
const deepCrawler = createDeepScanCrawler();
|
||||
const skillCrawler = createCrawler();
|
||||
|
||||
// Get repos that need scanning (never scanned or stale)
|
||||
const oneWeekAgo = new Date();
|
||||
oneWeekAgo.setDate(oneWeekAgo.getDate() - 7);
|
||||
const reposToScan = await discoveredRepoQueries.getNeedingScanning(deepDb, oneWeekAgo, 100);
|
||||
|
||||
if (reposToScan.length === 0) {
|
||||
console.log('No repositories need scanning');
|
||||
break;
|
||||
}
|
||||
|
||||
console.log(`Found ${reposToScan.length} repositories to scan`);
|
||||
|
||||
let scannedCount = 0;
|
||||
let skillsDiscovered = 0;
|
||||
let skillsIndexed = 0;
|
||||
|
||||
for (const repo of reposToScan) {
|
||||
try {
|
||||
console.log(`\nScanning ${repo.owner}/${repo.repo}...`);
|
||||
const skills = await deepCrawler.scanRepository(repo.owner, repo.repo);
|
||||
|
||||
// Mark as scanned
|
||||
await discoveredRepoQueries.markScanned(
|
||||
deepDb,
|
||||
repo.id,
|
||||
skills.length,
|
||||
skills.length > 0
|
||||
);
|
||||
|
||||
scannedCount++;
|
||||
skillsDiscovered += skills.length;
|
||||
|
||||
// If skills found, index them to the database
|
||||
if (skills.length > 0) {
|
||||
console.log(` Found ${skills.length} skills in ${repo.owner}/${repo.repo}, processing...`);
|
||||
let indexed = 0;
|
||||
let skipped = 0;
|
||||
let failed = 0;
|
||||
|
||||
for (const skillSource of skills) {
|
||||
try {
|
||||
const skillId = await indexSkill(
|
||||
skillCrawler,
|
||||
skillSource,
|
||||
false // don't force update if content unchanged
|
||||
);
|
||||
|
||||
if (skillId) {
|
||||
indexed++;
|
||||
console.log(` ✓ Indexed: ${skillSource.path}`);
|
||||
} else {
|
||||
skipped++;
|
||||
console.log(` → Skipped: ${skillSource.path} (unchanged or invalid)`);
|
||||
}
|
||||
} catch (error) {
|
||||
failed++;
|
||||
const errorMsg = error instanceof Error ? error.message : String(error);
|
||||
console.log(` ✗ Failed: ${skillSource.path} - ${errorMsg}`);
|
||||
}
|
||||
}
|
||||
|
||||
skillsIndexed += indexed;
|
||||
console.log(` Summary: ${indexed} indexed, ${skipped} skipped, ${failed} failed`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.log(` Error scanning: ${error}`);
|
||||
await discoveredRepoQueries.markScanned(deepDb, repo.id, 0, false, String(error));
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\nDeep scan complete:`);
|
||||
console.log(` Repositories scanned: ${scannedCount}`);
|
||||
console.log(` Skills discovered: ${skillsDiscovered}`);
|
||||
console.log(` Skills indexed: ${skillsIndexed}`);
|
||||
break;
|
||||
}
|
||||
|
||||
case 'multi-platform': {
|
||||
console.log('Searching for multi-platform instruction files...\n');
|
||||
const mpCrawler = createCrawler();
|
||||
|
||||
// Parse options
|
||||
const mpMaxPagesArg = process.argv.find(a => a.startsWith('--max-pages='));
|
||||
const mpMaxPages = mpMaxPagesArg ? parseInt(mpMaxPagesArg.split('=')[1]) : 10;
|
||||
const mpBudgetArg = process.argv.find(a => a.startsWith('--budget='));
|
||||
const mpBudgetPct = mpBudgetArg ? parseInt(mpBudgetArg.split('=')[1]) / 100 : 0.33;
|
||||
|
||||
// Only search for non-SKILL.md formats
|
||||
const nonSkillPatterns = INSTRUCTION_FILE_PATTERNS.filter(p => p.format !== 'skill.md');
|
||||
console.log(`Searching for ${nonSkillPatterns.length} formats: ${nonSkillPatterns.map(p => p.filename).join(', ')}`);
|
||||
console.log(`Options: max-pages=${mpMaxPages}, budget=${Math.round(mpBudgetPct * 100)}% reserve\n`);
|
||||
|
||||
// Check initial budget
|
||||
const initialBudget = await mpCrawler.checkBudget(mpBudgetPct);
|
||||
console.log(`API Budget: ${initialBudget.remaining}/${initialBudget.limit} remaining (reserve ${Math.round(mpBudgetPct * 100)}%)\n`);
|
||||
|
||||
const mpResults: Record<string, { discovered: number; indexed: number; failed: number; skipped: number }> = {};
|
||||
for (const pattern of nonSkillPatterns) {
|
||||
mpResults[pattern.format] = { discovered: 0, indexed: 0, failed: 0, skipped: 0 };
|
||||
}
|
||||
|
||||
// Search for each format with budget awareness
|
||||
for (const pattern of nonSkillPatterns) {
|
||||
// Check budget before each format
|
||||
const budget = await mpCrawler.checkBudget(mpBudgetPct);
|
||||
if (!budget.ok) {
|
||||
console.log(`\n⚠️ API budget low (${budget.remaining}/${budget.limit}). Waiting for reset...`);
|
||||
await mpCrawler.waitForBudget(mpBudgetPct);
|
||||
}
|
||||
|
||||
console.log(`\n═══ Searching for ${pattern.filename} ═══`);
|
||||
|
||||
try {
|
||||
const sources = await mpCrawler.searchGitHubForSkillsByFormat(
|
||||
pattern.format as SourceFormat,
|
||||
{ maxPages: mpMaxPages }
|
||||
);
|
||||
mpResults[pattern.format].discovered = sources.length;
|
||||
console.log(`Found ${sources.length} ${pattern.filename} files\n`);
|
||||
|
||||
// Index each discovered file (sequentially to respect API budget)
|
||||
for (const source of sources) {
|
||||
// Check budget periodically (every 20 skills)
|
||||
const formatStats = mpResults[pattern.format];
|
||||
const processed = formatStats.indexed + formatStats.skipped + formatStats.failed;
|
||||
if (processed > 0 && processed % 20 === 0) {
|
||||
const midBudget = await mpCrawler.checkBudget(mpBudgetPct);
|
||||
if (!midBudget.ok) {
|
||||
console.log(` ⚠️ Budget low mid-indexing (${midBudget.remaining}/${midBudget.limit}). Pausing...`);
|
||||
await mpCrawler.waitForBudget(mpBudgetPct);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const skillId = await indexSkill(mpCrawler, source, false);
|
||||
if (skillId) {
|
||||
mpResults[pattern.format].indexed++;
|
||||
} else {
|
||||
mpResults[pattern.format].skipped++;
|
||||
}
|
||||
} catch {
|
||||
mpResults[pattern.format].failed++;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error searching for ${pattern.filename}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
// Print summary
|
||||
console.log('\n' + '='.repeat(60));
|
||||
console.log('Multi-Platform Crawl Results');
|
||||
console.log('='.repeat(60));
|
||||
let totalDiscovered = 0;
|
||||
let totalIndexed = 0;
|
||||
for (const [format, stats] of Object.entries(mpResults)) {
|
||||
console.log(` ${format}: ${stats.discovered} discovered, ${stats.indexed} indexed, ${stats.skipped} skipped, ${stats.failed} failed`);
|
||||
totalDiscovered += stats.discovered;
|
||||
totalIndexed += stats.indexed;
|
||||
}
|
||||
console.log(` Total: ${totalDiscovered} discovered, ${totalIndexed} indexed`);
|
||||
|
||||
// Show final API status
|
||||
const finalBudget = await mpCrawler.checkBudget(mpBudgetPct);
|
||||
console.log(`\n API remaining: ${finalBudget.remaining}/${finalBudget.limit}`);
|
||||
console.log('='.repeat(60));
|
||||
break;
|
||||
}
|
||||
|
||||
case 'discovery-stats': {
|
||||
console.log('Discovery statistics:\n');
|
||||
const statsDb = createDb(process.env.DATABASE_URL);
|
||||
const repoStats = await discoveredRepoQueries.getStats(statsDb);
|
||||
const awesomeLists = await awesomeListQueries.getAll(statsDb);
|
||||
|
||||
console.log('Discovered Repositories:');
|
||||
console.log(` Total: ${repoStats.total}`);
|
||||
console.log(` Scanned: ${repoStats.scanned}`);
|
||||
console.log(` With skills: ${repoStats.withSkills}`);
|
||||
console.log('\nBy Discovery Source:');
|
||||
for (const source of repoStats.bySource) {
|
||||
console.log(` ${source.source}: ${source.count} repos (${source.withSkills} with skills)`);
|
||||
}
|
||||
|
||||
if (awesomeLists.length > 0) {
|
||||
console.log('\nAwesome Lists:');
|
||||
for (const list of awesomeLists) {
|
||||
const status = list.isActive ? 'active' : 'inactive';
|
||||
console.log(` ${list.id}: ${list.repoCount} repos (${status})`);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case 'full-enhanced': {
|
||||
console.log('Running full enhanced crawl (discovery + scan + index)...\n');
|
||||
const enhancedDb = createDb(process.env.DATABASE_URL);
|
||||
|
||||
// Step 1: Run all discovery strategies
|
||||
console.log('Step 1: Running discovery strategies...');
|
||||
const enhancedOrchestrator = createStrategyOrchestrator();
|
||||
const { repos: allDiscovered } = await enhancedOrchestrator.runAllStrategies();
|
||||
|
||||
console.log(`\nSaving ${allDiscovered.length} discovered repos...`);
|
||||
for (const repo of allDiscovered) {
|
||||
try {
|
||||
await discoveredRepoQueries.upsert(enhancedDb, {
|
||||
id: `${repo.owner}/${repo.repo}`,
|
||||
owner: repo.owner,
|
||||
repo: repo.repo,
|
||||
discoveredVia: repo.discoveredVia,
|
||||
githubStars: repo.stars,
|
||||
});
|
||||
} catch {
|
||||
// Skip
|
||||
}
|
||||
}
|
||||
|
||||
// Step 2: Schedule full crawl (uses existing crawler with code search)
|
||||
console.log('\nStep 2: Scheduling full crawl...');
|
||||
const enhancedJobId = await scheduleFullCrawl({
|
||||
minStars: parseInt(process.env.INDEXER_MIN_STARS || '0'),
|
||||
});
|
||||
console.log(`Full crawl scheduled with job ID: ${enhancedJobId}`);
|
||||
|
||||
console.log('\nFull enhanced crawl initiated. Monitor with: pnpm crawl stats');
|
||||
break;
|
||||
}
|
||||
|
||||
case 'process-add-requests': {
|
||||
console.log('Processing pending add requests...\n');
|
||||
const addDb = createDb(process.env.DATABASE_URL);
|
||||
const addCrawler = createCrawler();
|
||||
|
||||
// Check queue status first
|
||||
const queueStats = await getQueueStats();
|
||||
if (queueStats.active > 0) {
|
||||
console.log(`⚠️ Warning: ${queueStats.active} active jobs in queue`);
|
||||
console.log(' Processing add requests may be slower due to concurrent operations.\n');
|
||||
}
|
||||
|
||||
// Check GitHub rate limit
|
||||
try {
|
||||
const rateLimitInfo = await addCrawler.getRateLimitStatus();
|
||||
console.log(`GitHub API: ${rateLimitInfo.remaining}/${rateLimitInfo.limit} requests remaining`);
|
||||
if (rateLimitInfo.remaining < 100) {
|
||||
const resetTime = new Date(rateLimitInfo.resetAt);
|
||||
console.log(`⚠️ Warning: Low API quota. Resets at ${resetTime.toLocaleTimeString()}`);
|
||||
if (rateLimitInfo.remaining < 10) {
|
||||
console.log('❌ Aborting: Not enough API quota. Try again after reset.');
|
||||
break;
|
||||
}
|
||||
}
|
||||
console.log('');
|
||||
} catch {
|
||||
console.log('⚠️ Could not check GitHub rate limit. Proceeding anyway...\n');
|
||||
}
|
||||
|
||||
// Get all pending add requests
|
||||
const pendingRequests = await addRequestQueries.getAllPending(addDb);
|
||||
|
||||
if (pendingRequests.length === 0) {
|
||||
console.log('✓ No pending add requests found');
|
||||
break;
|
||||
}
|
||||
|
||||
console.log(`Found ${pendingRequests.length} pending request(s)\n`);
|
||||
|
||||
let processedCount = 0;
|
||||
let skillsIndexed = 0;
|
||||
let skillsAlreadyIndexed = 0;
|
||||
let skillsSkipped = 0;
|
||||
let failedCount = 0;
|
||||
let rateLimitHit = false;
|
||||
|
||||
for (const request of pendingRequests) {
|
||||
// Stop if we hit rate limit
|
||||
if (rateLimitHit) {
|
||||
console.log(`⏳ Rate limit hit. Remaining requests will be processed later.`);
|
||||
break;
|
||||
}
|
||||
|
||||
console.log(`Processing request ${request.id.slice(0, 8)}...`);
|
||||
console.log(` Repository: ${request.repositoryUrl}`);
|
||||
console.log(` Skill paths: ${request.skillPath || 'auto-detect'}`);
|
||||
|
||||
// Skip if no skill paths found
|
||||
if (!request.hasSkillMd || !request.skillPath) {
|
||||
console.log(' → No SKILL.md paths found, marking for manual review');
|
||||
await addRequestQueries.updateStatus(addDb, request.id, {
|
||||
status: 'approved',
|
||||
errorMessage: 'No SKILL.md found - requires manual review',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
// Parse repository URL
|
||||
const url = new URL(request.repositoryUrl);
|
||||
const pathParts = url.pathname.split('/').filter(Boolean);
|
||||
const owner = pathParts[0];
|
||||
const repo = pathParts[1];
|
||||
|
||||
if (!owner || !repo) {
|
||||
throw new Error('Invalid repository URL');
|
||||
}
|
||||
|
||||
// Get default branch
|
||||
const repoMeta = await addCrawler.getRepoMetadata(owner, repo);
|
||||
const branch = repoMeta.defaultBranch;
|
||||
|
||||
// Parse skill paths (comma-separated)
|
||||
const skillPaths = request.skillPath.split(',').map((p: string) => p.trim());
|
||||
const indexedSkillIds: string[] = [];
|
||||
const existingSkillIds: string[] = [];
|
||||
|
||||
for (const skillPath of skillPaths) {
|
||||
try {
|
||||
// Check if skill already exists before indexing
|
||||
const skillName = skillPath.split('/').pop() || 'skill';
|
||||
const potentialSkillId = `${owner}/${repo}/${skillName}`;
|
||||
const existingSkill = await skillQueries.getById(addDb, potentialSkillId);
|
||||
|
||||
if (existingSkill && !existingSkill.isBlocked) {
|
||||
// Skill already indexed
|
||||
existingSkillIds.push(potentialSkillId);
|
||||
skillsAlreadyIndexed++;
|
||||
console.log(` ${skillPath || 'root'}: Already indexed ✓`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// If skill is blocked, check if requester is the owner
|
||||
if (existingSkill && existingSkill.isBlocked) {
|
||||
// Get requester's GitHub username
|
||||
const requester = await userQueries.getById(addDb, request.userId);
|
||||
const requesterUsername = requester?.username?.toLowerCase();
|
||||
const repoOwner = owner.toLowerCase();
|
||||
|
||||
if (requesterUsername && requesterUsername === repoOwner) {
|
||||
// Owner is re-adding their skill, unblock it
|
||||
console.log(` ${skillPath || 'root'}: Unblocking (owner re-add request)...`);
|
||||
await skillQueries.unblock(addDb, potentialSkillId);
|
||||
existingSkillIds.push(potentialSkillId);
|
||||
skillsAlreadyIndexed++;
|
||||
console.log(` → Unblocked: ${potentialSkillId}`);
|
||||
continue;
|
||||
} else {
|
||||
// Not the owner, skip blocked skill
|
||||
console.log(` ${skillPath || 'root'}: Blocked by owner, skipping`);
|
||||
skillsSkipped++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(` ${skillPath || 'root'}: Indexing...`);
|
||||
const skillId = await indexSkill(
|
||||
addCrawler,
|
||||
{
|
||||
owner,
|
||||
repo,
|
||||
path: skillPath || '.',
|
||||
branch,
|
||||
},
|
||||
true // force update
|
||||
);
|
||||
|
||||
if (skillId) {
|
||||
indexedSkillIds.push(skillId);
|
||||
skillsIndexed++;
|
||||
console.log(` → Indexed: ${skillId}`);
|
||||
} else {
|
||||
skillsSkipped++;
|
||||
console.log(` → Skipped (invalid skill)`);
|
||||
}
|
||||
} catch (skillError: unknown) {
|
||||
// Check for rate limit error
|
||||
const errorMessage = skillError instanceof Error ? skillError.message : String(skillError);
|
||||
if (errorMessage.includes('rate limit') || errorMessage.includes('403')) {
|
||||
rateLimitHit = true;
|
||||
console.log(` → Rate limit hit, will retry later`);
|
||||
break;
|
||||
}
|
||||
console.error(` → Failed: ${errorMessage}`);
|
||||
skillsSkipped++;
|
||||
}
|
||||
}
|
||||
|
||||
// Combine all skill IDs (new + existing)
|
||||
const allSkillIds = [...indexedSkillIds, ...existingSkillIds];
|
||||
|
||||
// Update request status
|
||||
if (allSkillIds.length > 0) {
|
||||
await addRequestQueries.updateStatus(addDb, request.id, {
|
||||
status: 'indexed',
|
||||
indexedSkillId: allSkillIds.join(','),
|
||||
});
|
||||
const newCount = indexedSkillIds.length;
|
||||
const existingCount = existingSkillIds.length;
|
||||
if (newCount > 0 && existingCount > 0) {
|
||||
console.log(` ✓ Indexed ${newCount} new + ${existingCount} already existed`);
|
||||
} else if (newCount > 0) {
|
||||
console.log(` ✓ Indexed ${newCount} skill(s)`);
|
||||
} else {
|
||||
console.log(` ✓ All ${existingCount} skill(s) were already indexed`);
|
||||
}
|
||||
} else if (rateLimitHit) {
|
||||
// Leave as pending if we hit rate limit
|
||||
console.log(` ⏳ Left as pending (rate limit)`);
|
||||
} else {
|
||||
await addRequestQueries.updateStatus(addDb, request.id, {
|
||||
status: 'rejected',
|
||||
errorMessage: 'No valid skills could be indexed',
|
||||
});
|
||||
console.log(` ✗ Rejected (no valid skills)`);
|
||||
}
|
||||
|
||||
processedCount++;
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
|
||||
|
||||
// Check for rate limit error
|
||||
if (errorMessage.includes('rate limit') || errorMessage.includes('403')) {
|
||||
rateLimitHit = true;
|
||||
console.log(` ⏳ Rate limit hit, will retry later`);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.error(` ✗ Error: ${errorMessage}`);
|
||||
failedCount++;
|
||||
|
||||
await addRequestQueries.updateStatus(addDb, request.id, {
|
||||
status: 'rejected',
|
||||
errorMessage,
|
||||
});
|
||||
}
|
||||
console.log(''); // Empty line between requests
|
||||
}
|
||||
|
||||
console.log(`═══════════════════════════════════════`);
|
||||
console.log(`Processing Summary:`);
|
||||
console.log(` Requests processed: ${processedCount}`);
|
||||
console.log(` Skills newly indexed: ${skillsIndexed}`);
|
||||
console.log(` Skills already indexed: ${skillsAlreadyIndexed}`);
|
||||
console.log(` Skills skipped/invalid: ${skillsSkipped}`);
|
||||
console.log(` Requests failed: ${failedCount}`);
|
||||
if (rateLimitHit) {
|
||||
console.log(`\n⚠️ Rate limit reached. Some requests were left pending.`);
|
||||
console.log(` Run 'process-add-requests' again after GitHub rate limit resets.`);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
default:
|
||||
console.log('Usage: pnpm crawl <command>\n');
|
||||
console.log('Basic Commands:');
|
||||
console.log(' full - Schedule a full crawl of all skills');
|
||||
console.log(' incremental - Schedule an incremental crawl (last 24h)');
|
||||
console.log(' stats - Show queue statistics\n');
|
||||
console.log('Enhanced Discovery:');
|
||||
console.log(' full-enhanced - Full crawl with all discovery strategies');
|
||||
console.log(' discover-repos - Run all discovery strategies (awesome lists, topics, forks)');
|
||||
console.log(' awesome-lists - Crawl awesome lists for repo discovery');
|
||||
console.log(' deep-scan - Deep scan discovered repos for SKILL.md files');
|
||||
console.log(' multi-platform - Search for non-SKILL.md formats (--max-pages=N --budget=N)');
|
||||
console.log(' discovery-stats - Show discovery statistics\n');
|
||||
console.log('Maintenance:');
|
||||
console.log(' sync-meili - Sync all skills from database to Meilisearch');
|
||||
console.log(' recategorize - Re-categorize all skills with 23 categories (alias: link-categories)');
|
||||
console.log(' process-add-requests - Process pending add requests (auto-index skills)');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error('Error:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
883
services/indexer/src/crawler.ts
Normal file
883
services/indexer/src/crawler.ts
Normal file
@@ -0,0 +1,883 @@
|
||||
import type { Octokit } from '@octokit/rest';
|
||||
import { INSTRUCTION_FILE_PATTERNS, type SkillSource, type SourceFormat } from 'skillhub-core';
|
||||
import { TokenManager } from './token-manager.js';
|
||||
import { OctokitPool } from './octokit-pool.js';
|
||||
|
||||
export interface DiscoverOptions {
|
||||
minStars?: number;
|
||||
language?: string;
|
||||
updatedAfter?: Date;
|
||||
perPage?: number;
|
||||
maxPages?: number;
|
||||
}
|
||||
|
||||
export interface SkillContent {
|
||||
skillMd: string;
|
||||
files: FileInfo[];
|
||||
scripts: ScriptFile[];
|
||||
references: ReferenceFile[];
|
||||
repoMeta: RepoMetadata;
|
||||
}
|
||||
|
||||
export interface FileInfo {
|
||||
name: string;
|
||||
path: string;
|
||||
type: 'file' | 'dir';
|
||||
size?: number;
|
||||
}
|
||||
|
||||
export interface ScriptFile {
|
||||
name: string;
|
||||
path: string;
|
||||
content: string;
|
||||
language?: string;
|
||||
}
|
||||
|
||||
export interface ReferenceFile {
|
||||
name: string;
|
||||
path: string;
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface RepoMetadata {
|
||||
stars: number;
|
||||
forks: number;
|
||||
license: string | null;
|
||||
description: string | null;
|
||||
updatedAt: string;
|
||||
defaultBranch: string;
|
||||
topics: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Official skill sources that are always crawled
|
||||
*/
|
||||
const OFFICIAL_SKILL_SOURCES = [
|
||||
{ owner: 'anthropics', repo: 'skills', skillsPath: 'skills' },
|
||||
{ owner: 'anthropics', repo: 'claude-code', skillsPath: 'skills' },
|
||||
];
|
||||
|
||||
/**
|
||||
* Known community skill repositories
|
||||
* Add more as they become available
|
||||
*/
|
||||
const COMMUNITY_SKILL_SOURCES = [
|
||||
{ owner: 'obra', repo: 'superpowers', skillsPath: 'skills' },
|
||||
// openclaw/skills - Clawdhub archive with 2200+ skills (303 stars)
|
||||
{ owner: 'openclaw', repo: 'skills', skillsPath: 'skills' },
|
||||
// Add more community sources here as they become available
|
||||
];
|
||||
|
||||
/**
|
||||
* Common paths where skills are stored in repositories
|
||||
* Note: Used by deep-scan strategy to know where to look for skills
|
||||
* Exported so other modules can use this configuration
|
||||
*/
|
||||
export const SKILL_PATHS = [
|
||||
'skills',
|
||||
'.claude/skills',
|
||||
'.github/skills',
|
||||
'.codex/skills',
|
||||
];
|
||||
|
||||
/**
|
||||
* GitHub Crawler for discovering and fetching Agent Skills
|
||||
*/
|
||||
export class GitHubCrawler {
|
||||
private octokitPool: OctokitPool;
|
||||
private tokenManager: TokenManager;
|
||||
private lastCodeSearchTime = 0;
|
||||
|
||||
// GitHub Code Search secondary rate limit: ~10 req/min per token
|
||||
// Use 7s delay between requests for safety margin
|
||||
private static readonly CODE_SEARCH_DELAY_MS = 7000;
|
||||
|
||||
constructor(tokenManager?: TokenManager) {
|
||||
this.tokenManager = tokenManager || TokenManager.getInstance();
|
||||
this.octokitPool = new OctokitPool(this.tokenManager);
|
||||
}
|
||||
|
||||
/**
|
||||
* Enforce code search secondary rate limit delay
|
||||
*/
|
||||
private async waitForCodeSearchSlot(): Promise<void> {
|
||||
const elapsed = Date.now() - this.lastCodeSearchTime;
|
||||
const delay = GitHubCrawler.CODE_SEARCH_DELAY_MS - elapsed;
|
||||
if (delay > 0) {
|
||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||
}
|
||||
this.lastCodeSearchTime = Date.now();
|
||||
}
|
||||
|
||||
private async getOctokit(): Promise<Octokit> {
|
||||
return this.octokitPool.getBestInstance();
|
||||
}
|
||||
|
||||
private getCurrentToken(): string {
|
||||
return this.tokenManager.getBestToken();
|
||||
}
|
||||
|
||||
/**
|
||||
* Discover skills from all sources: official repos, community repos, and GitHub search
|
||||
*/
|
||||
async discoverSkillRepos(options: DiscoverOptions = {}): Promise<SkillSource[]> {
|
||||
const results: SkillSource[] = [];
|
||||
|
||||
// 1. Fetch from official Anthropic skills repository
|
||||
console.log('Fetching skills from official sources...');
|
||||
for (const source of OFFICIAL_SKILL_SOURCES) {
|
||||
try {
|
||||
const skills = await this.fetchSkillsFromRepo(source.owner, source.repo, source.skillsPath);
|
||||
console.log(`Found ${skills.length} skills in ${source.owner}/${source.repo}`);
|
||||
results.push(...skills);
|
||||
} catch (error) {
|
||||
console.warn(`Failed to fetch from ${source.owner}/${source.repo}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Fetch from known community repositories
|
||||
console.log('Fetching skills from community sources...');
|
||||
for (const source of COMMUNITY_SKILL_SOURCES) {
|
||||
try {
|
||||
const skills = await this.fetchSkillsFromRepo(source.owner, source.repo, source.skillsPath);
|
||||
console.log(`Found ${skills.length} skills in ${source.owner}/${source.repo}`);
|
||||
results.push(...skills);
|
||||
} catch (error) {
|
||||
console.warn(`Failed to fetch from ${source.owner}/${source.repo}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Search GitHub for other repositories with SKILL.md files
|
||||
console.log('Searching GitHub for additional skills...');
|
||||
const searchResults = await this.searchGitHubForSkills(options);
|
||||
console.log(`Found ${searchResults.length} skills from GitHub search`);
|
||||
results.push(...searchResults);
|
||||
|
||||
return this.deduplicateResults(results);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch all skills from a specific repository's skills directory
|
||||
*/
|
||||
async fetchSkillsFromRepo(owner: string, repo: string, skillsPath: string): Promise<SkillSource[]> {
|
||||
const results: SkillSource[] = [];
|
||||
|
||||
try {
|
||||
// Get repository metadata for default branch
|
||||
const repoMeta = await this.getRepoMetadata(owner, repo);
|
||||
const branch = repoMeta.defaultBranch;
|
||||
|
||||
// List contents of skills directory
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const response = await octokit.repos.getContent({
|
||||
owner,
|
||||
repo,
|
||||
path: skillsPath,
|
||||
ref: branch,
|
||||
});
|
||||
|
||||
this.octokitPool.updateStats(token, response.headers);
|
||||
|
||||
if (!Array.isArray(response.data)) {
|
||||
// Single file or not found - check if it's a SKILL.md at root
|
||||
if (skillsPath === '.' || skillsPath === '') {
|
||||
const hasSkillMd = await this.checkFileExists(owner, repo, 'SKILL.md', branch);
|
||||
if (hasSkillMd) {
|
||||
results.push({ owner, repo, path: '.', branch });
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
// Check each subdirectory for SKILL.md
|
||||
for (const item of response.data) {
|
||||
if (item.type === 'dir') {
|
||||
const skillMdPath = `${skillsPath}/${item.name}/SKILL.md`;
|
||||
const hasSkillMd = await this.checkFileExists(owner, repo, skillMdPath, branch);
|
||||
|
||||
if (hasSkillMd) {
|
||||
results.push({
|
||||
owner,
|
||||
repo,
|
||||
path: `${skillsPath}/${item.name}`,
|
||||
branch,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
} catch (error) {
|
||||
if (this.isNotFoundError(error)) {
|
||||
return [];
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a file exists in a repository
|
||||
*/
|
||||
private async checkFileExists(owner: string, repo: string, path: string, ref: string): Promise<boolean> {
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const response = await octokit.repos.getContent({
|
||||
owner,
|
||||
repo,
|
||||
path,
|
||||
ref,
|
||||
});
|
||||
this.octokitPool.updateStats(token, response.headers);
|
||||
return 'content' in response.data;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* All search strategies for GitHub Code Search
|
||||
*/
|
||||
private getSearchStrategies(): Array<{ label: string; query: string; format: SourceFormat }> {
|
||||
return [
|
||||
// === SKILL.md strategies ===
|
||||
{ label: 'all-skills', query: 'filename:SKILL.md', format: 'skill.md' },
|
||||
{ label: 'skills-folder', query: 'filename:SKILL.md path:skills', format: 'skill.md' },
|
||||
{ label: 'claude-folder', query: 'filename:SKILL.md path:.claude', format: 'skill.md' },
|
||||
{ label: 'github-folder', query: 'filename:SKILL.md path:.github', format: 'skill.md' },
|
||||
{ label: 'codex-folder', query: 'filename:SKILL.md path:.codex', format: 'skill.md' },
|
||||
{ label: 'small-files', query: 'filename:SKILL.md size:<1000', format: 'skill.md' },
|
||||
{ label: 'medium-files', query: 'filename:SKILL.md size:1000..5000', format: 'skill.md' },
|
||||
{ label: 'large-files', query: 'filename:SKILL.md size:>5000', format: 'skill.md' },
|
||||
// === AGENTS.md strategies (Codex) ===
|
||||
{ label: 'agents-md', query: 'filename:AGENTS.md', format: 'agents.md' },
|
||||
{ label: 'agents-md-sized', query: 'filename:AGENTS.md size:>200', format: 'agents.md' },
|
||||
// === .cursorrules strategies ===
|
||||
{ label: 'cursorrules', query: 'filename:.cursorrules', format: 'cursorrules' },
|
||||
{ label: 'cursorrules-sized', query: 'filename:.cursorrules size:>200', format: 'cursorrules' },
|
||||
// === .windsurfrules strategies ===
|
||||
{ label: 'windsurfrules', query: 'filename:.windsurfrules', format: 'windsurfrules' },
|
||||
// === copilot-instructions.md strategies ===
|
||||
{ label: 'copilot-instructions', query: 'filename:copilot-instructions.md path:.github', format: 'copilot-instructions' },
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Search GitHub for repositories containing SKILL.md files
|
||||
* Uses multiple search strategies to get more results
|
||||
*/
|
||||
async searchGitHubForSkills(options: DiscoverOptions = {}): Promise<SkillSource[]> {
|
||||
const { minStars = 0, perPage = 100, maxPages = 10 } = options;
|
||||
|
||||
const allResults: SkillSource[] = [];
|
||||
const searchStrategies = this.getSearchStrategies();
|
||||
|
||||
console.log(`Starting multi-strategy GitHub search (${searchStrategies.length} strategies)...`);
|
||||
|
||||
for (const strategy of searchStrategies) {
|
||||
console.log(`\n🔍 Strategy: ${strategy.label}`);
|
||||
const strategyResults = await this.runSegmentedSearch(
|
||||
strategy.query,
|
||||
strategy.label,
|
||||
{ perPage, maxPages, minStars },
|
||||
strategy.format
|
||||
);
|
||||
allResults.push(...strategyResults);
|
||||
|
||||
// Deduplicate after each strategy to see actual progress
|
||||
const currentUnique = this.deduplicateResults(allResults);
|
||||
console.log(` → Found ${strategyResults.length} (unique total: ${currentUnique.length})`);
|
||||
}
|
||||
|
||||
// Final deduplicate
|
||||
const deduplicated = this.deduplicateResults(allResults);
|
||||
console.log(`\n✅ Total unique skills from GitHub search: ${deduplicated.length}`);
|
||||
|
||||
return deduplicated;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a single segmented search query
|
||||
*/
|
||||
private async runSegmentedSearch(
|
||||
query: string,
|
||||
segmentLabel: string,
|
||||
options: { perPage: number; maxPages: number; minStars: number },
|
||||
expectedFormat: SourceFormat = 'skill.md'
|
||||
): Promise<SkillSource[]> {
|
||||
const { perPage, maxPages } = options;
|
||||
const results: SkillSource[] = [];
|
||||
let page = 1;
|
||||
|
||||
// Resolve expected filename from the format
|
||||
const pattern = INSTRUCTION_FILE_PATTERNS.find(p => p.format === expectedFormat);
|
||||
const expectedFilename = pattern?.filename || 'SKILL.md';
|
||||
|
||||
while (page <= maxPages) {
|
||||
try {
|
||||
// Enforce code search secondary rate limit delay
|
||||
await this.waitForCodeSearchSlot();
|
||||
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const response = await octokit.search.code({
|
||||
q: query,
|
||||
per_page: perPage,
|
||||
page,
|
||||
});
|
||||
|
||||
this.octokitPool.updateStats(token, response.headers);
|
||||
|
||||
if (page === 1) {
|
||||
console.log(` Query: ${query}`);
|
||||
console.log(` Total available: ${response.data.total_count}`);
|
||||
}
|
||||
|
||||
if (response.data.items.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
for (const item of response.data.items) {
|
||||
if (item.name !== expectedFilename) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Extract path: strip the filename to get the directory
|
||||
const escapedFilename = expectedFilename.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const skillPath = item.path
|
||||
.replace(new RegExp(`/?${escapedFilename}$`), '') || '.';
|
||||
|
||||
results.push({
|
||||
owner: item.repository.owner.login,
|
||||
repo: item.repository.name,
|
||||
path: skillPath,
|
||||
branch: 'main',
|
||||
sourceFormat: expectedFormat,
|
||||
});
|
||||
}
|
||||
|
||||
if (response.data.items.length < perPage) {
|
||||
break;
|
||||
}
|
||||
|
||||
page++;
|
||||
} catch (error) {
|
||||
if (this.isSecondaryRateLimitError(error)) {
|
||||
// Secondary rate limit (code search) - wait and retry
|
||||
const retryAfter = this.getRetryAfterSeconds(error);
|
||||
console.log(` ⏳ Code search rate limit, waiting ${retryAfter}s...`);
|
||||
await new Promise((resolve) => setTimeout(resolve, retryAfter * 1000));
|
||||
continue;
|
||||
}
|
||||
if (this.isRateLimitError(error)) {
|
||||
await this.waitForRateLimit();
|
||||
continue;
|
||||
}
|
||||
if (this.isBeyondResultsLimit(error)) {
|
||||
console.log(` ⚠️ Reached 1000 limit for segment ${segmentLabel}`);
|
||||
break;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a specific skill's content from GitHub
|
||||
*/
|
||||
async fetchSkillContent(source: SkillSource): Promise<SkillContent> {
|
||||
// Rate limit is now handled by OctokitPool automatically
|
||||
const { owner, repo, path, branch } = source;
|
||||
const sourceFormat = source.sourceFormat || 'skill.md';
|
||||
|
||||
// Get repository metadata
|
||||
const repoMeta = await this.getRepoMetadata(owner, repo);
|
||||
const actualBranch = branch || repoMeta.defaultBranch;
|
||||
|
||||
// Determine file path based on format
|
||||
const pattern = INSTRUCTION_FILE_PATTERNS.find(p => p.format === sourceFormat);
|
||||
const filename = pattern?.filename || 'SKILL.md';
|
||||
let filePath: string;
|
||||
|
||||
if (sourceFormat === 'copilot-instructions') {
|
||||
filePath = '.github/copilot-instructions.md';
|
||||
} else if (sourceFormat === 'cursorrules' || sourceFormat === 'windsurfrules') {
|
||||
filePath = filename; // Root-only files
|
||||
} else {
|
||||
filePath = path === '.' ? filename : `${path}/${filename}`;
|
||||
}
|
||||
|
||||
const skillMd = await this.fetchFileContent(owner, repo, filePath, actualBranch);
|
||||
|
||||
// For non-SKILL.md formats, don't look for scripts/references subdirectories
|
||||
const isStandaloneFormat = sourceFormat !== 'skill.md';
|
||||
|
||||
const files = isStandaloneFormat
|
||||
? []
|
||||
: await this.listDirectory(owner, repo, path === '.' ? '' : path, actualBranch);
|
||||
|
||||
const scripts = isStandaloneFormat
|
||||
? []
|
||||
: await this.fetchScripts(owner, repo, path, actualBranch, files);
|
||||
|
||||
const references = isStandaloneFormat
|
||||
? []
|
||||
: await this.fetchReferences(owner, repo, path, actualBranch, files);
|
||||
|
||||
return {
|
||||
skillMd,
|
||||
files,
|
||||
scripts,
|
||||
references,
|
||||
repoMeta,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get repository metadata
|
||||
*/
|
||||
async getRepoMetadata(owner: string, repo: string): Promise<RepoMetadata> {
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const response = await octokit.repos.get({ owner, repo });
|
||||
this.octokitPool.updateStats(token, response.headers);
|
||||
|
||||
return {
|
||||
stars: response.data.stargazers_count,
|
||||
forks: response.data.forks_count,
|
||||
license: response.data.license?.spdx_id || null,
|
||||
description: response.data.description,
|
||||
updatedAt: response.data.updated_at,
|
||||
defaultBranch: response.data.default_branch,
|
||||
topics: response.data.topics || [],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a GitHub user's public email address
|
||||
* First tries Profile API, then falls back to latest commit email
|
||||
*/
|
||||
async fetchOwnerEmail(username: string): Promise<{ email: string | null; source: string | null }> {
|
||||
try {
|
||||
// Step 1: GitHub Profile API
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const userResponse = await octokit.users.getByUsername({ username });
|
||||
this.octokitPool.updateStats(token, userResponse.headers);
|
||||
|
||||
if (userResponse.data.email) {
|
||||
return { email: userResponse.data.email, source: 'profile' };
|
||||
}
|
||||
|
||||
// Step 2: Fallback — check recent commit emails from their repos
|
||||
try {
|
||||
const reposResponse = await octokit.repos.listForUser({
|
||||
username,
|
||||
sort: 'pushed',
|
||||
per_page: 1,
|
||||
});
|
||||
this.octokitPool.updateStats(token, reposResponse.headers);
|
||||
|
||||
if (reposResponse.data.length > 0) {
|
||||
const repo = reposResponse.data[0];
|
||||
const commitsResponse = await octokit.repos.listCommits({
|
||||
owner: username,
|
||||
repo: repo.name,
|
||||
per_page: 5,
|
||||
author: username,
|
||||
});
|
||||
this.octokitPool.updateStats(token, commitsResponse.headers);
|
||||
|
||||
for (const commit of commitsResponse.data) {
|
||||
const email = commit.commit.author?.email;
|
||||
if (
|
||||
email &&
|
||||
!email.includes('noreply') &&
|
||||
!email.includes('users.noreply.github.com') &&
|
||||
email.includes('@')
|
||||
) {
|
||||
return { email, source: 'commit' };
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Fallback failed — not critical
|
||||
}
|
||||
|
||||
return { email: null, source: null };
|
||||
} catch (error) {
|
||||
console.warn(`[Crawler] Could not fetch email for ${username}:`, error instanceof Error ? error.message : error);
|
||||
return { email: null, source: null };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a single file's content
|
||||
*/
|
||||
private async fetchFileContent(
|
||||
owner: string,
|
||||
repo: string,
|
||||
path: string,
|
||||
ref: string
|
||||
): Promise<string> {
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const response = await octokit.repos.getContent({
|
||||
owner,
|
||||
repo,
|
||||
path,
|
||||
ref,
|
||||
});
|
||||
|
||||
this.octokitPool.updateStats(token, response.headers);
|
||||
|
||||
if ('content' in response.data && response.data.type === 'file') {
|
||||
return Buffer.from(response.data.content, 'base64').toString('utf-8');
|
||||
}
|
||||
|
||||
throw new Error(`Path ${path} is not a file`);
|
||||
} catch (error) {
|
||||
if (this.isNotFoundError(error)) {
|
||||
throw new Error(`File not found: ${path}`);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List files in a directory
|
||||
*/
|
||||
private async listDirectory(
|
||||
owner: string,
|
||||
repo: string,
|
||||
path: string,
|
||||
ref: string
|
||||
): Promise<FileInfo[]> {
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const response = await octokit.repos.getContent({
|
||||
owner,
|
||||
repo,
|
||||
path: path || '.',
|
||||
ref,
|
||||
});
|
||||
|
||||
this.octokitPool.updateStats(token, response.headers);
|
||||
|
||||
if (!Array.isArray(response.data)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return response.data.map((item) => ({
|
||||
name: item.name,
|
||||
path: item.path,
|
||||
type: item.type as 'file' | 'dir',
|
||||
size: item.size,
|
||||
}));
|
||||
} catch (error) {
|
||||
if (this.isNotFoundError(error)) {
|
||||
return [];
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch all scripts from the skill
|
||||
*/
|
||||
private async fetchScripts(
|
||||
owner: string,
|
||||
repo: string,
|
||||
basePath: string,
|
||||
ref: string,
|
||||
files: FileInfo[]
|
||||
): Promise<ScriptFile[]> {
|
||||
const scripts: ScriptFile[] = [];
|
||||
const scriptsDir = files.find((f) => f.name === 'scripts' && f.type === 'dir');
|
||||
|
||||
if (!scriptsDir) {
|
||||
return scripts;
|
||||
}
|
||||
|
||||
const scriptPath = basePath === '.' ? 'scripts' : `${basePath}/scripts`;
|
||||
const scriptFiles = await this.listDirectory(owner, repo, scriptPath, ref);
|
||||
|
||||
const scriptExtensions = ['.sh', '.bash', '.py', '.js', '.ts', '.rb', '.ps1'];
|
||||
|
||||
for (const file of scriptFiles) {
|
||||
if (file.type !== 'file') continue;
|
||||
|
||||
const ext = file.name.substring(file.name.lastIndexOf('.'));
|
||||
if (!scriptExtensions.includes(ext)) continue;
|
||||
|
||||
try {
|
||||
const content = await this.fetchFileContent(owner, repo, file.path, ref);
|
||||
scripts.push({
|
||||
name: file.name,
|
||||
path: file.path,
|
||||
content,
|
||||
language: this.getLanguageFromExtension(ext),
|
||||
});
|
||||
} catch {
|
||||
// Skip files that can't be fetched
|
||||
}
|
||||
}
|
||||
|
||||
return scripts;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch all reference files from the skill
|
||||
*/
|
||||
private async fetchReferences(
|
||||
owner: string,
|
||||
repo: string,
|
||||
basePath: string,
|
||||
ref: string,
|
||||
files: FileInfo[]
|
||||
): Promise<ReferenceFile[]> {
|
||||
const references: ReferenceFile[] = [];
|
||||
const refsDir = files.find((f) => f.name === 'references' && f.type === 'dir');
|
||||
|
||||
if (!refsDir) {
|
||||
return references;
|
||||
}
|
||||
|
||||
const refPath = basePath === '.' ? 'references' : `${basePath}/references`;
|
||||
const refFiles = await this.listDirectory(owner, repo, refPath, ref);
|
||||
|
||||
// Only fetch text-based reference files
|
||||
const textExtensions = ['.md', '.txt', '.json', '.yaml', '.yml', '.xml', '.html', '.css'];
|
||||
|
||||
for (const file of refFiles) {
|
||||
if (file.type !== 'file') continue;
|
||||
if (file.size && file.size > 100000) continue; // Skip large files
|
||||
|
||||
const ext = file.name.substring(file.name.lastIndexOf('.'));
|
||||
if (!textExtensions.includes(ext)) continue;
|
||||
|
||||
try {
|
||||
const content = await this.fetchFileContent(owner, repo, file.path, ref);
|
||||
references.push({
|
||||
name: file.name,
|
||||
path: file.path,
|
||||
content,
|
||||
});
|
||||
} catch {
|
||||
// Skip files that can't be fetched
|
||||
}
|
||||
}
|
||||
|
||||
return references;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get programming language from file extension
|
||||
*/
|
||||
private getLanguageFromExtension(ext: string): string {
|
||||
const languages: Record<string, string> = {
|
||||
'.sh': 'bash',
|
||||
'.bash': 'bash',
|
||||
'.py': 'python',
|
||||
'.js': 'javascript',
|
||||
'.ts': 'typescript',
|
||||
'.rb': 'ruby',
|
||||
'.ps1': 'powershell',
|
||||
};
|
||||
return languages[ext] || 'unknown';
|
||||
}
|
||||
|
||||
/**
|
||||
* Search GitHub for skills of a specific format (e.g., cursorrules, agents.md)
|
||||
* Used by multi-platform crawl command to search one format at a time
|
||||
*/
|
||||
async searchGitHubForSkillsByFormat(
|
||||
format: SourceFormat,
|
||||
options: DiscoverOptions = {}
|
||||
): Promise<SkillSource[]> {
|
||||
const { perPage = 100, maxPages = 10, minStars = 0 } = options;
|
||||
const allResults: SkillSource[] = [];
|
||||
|
||||
const matchingStrategies = this.getSearchStrategies().filter(s => s.format === format);
|
||||
if (matchingStrategies.length === 0) {
|
||||
console.log(`No search strategies defined for format: ${format}`);
|
||||
return [];
|
||||
}
|
||||
|
||||
console.log(`Searching for ${format} files (${matchingStrategies.length} strategies)...`);
|
||||
|
||||
for (const strategy of matchingStrategies) {
|
||||
console.log(`\n Strategy: ${strategy.label}`);
|
||||
const results = await this.runSegmentedSearch(
|
||||
strategy.query,
|
||||
strategy.label,
|
||||
{ perPage, maxPages, minStars },
|
||||
strategy.format
|
||||
);
|
||||
allResults.push(...results);
|
||||
}
|
||||
|
||||
const deduplicated = this.deduplicateResults(allResults);
|
||||
console.log(`Found ${deduplicated.length} unique ${format} files`);
|
||||
return deduplicated;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deduplicate skill sources
|
||||
*/
|
||||
private deduplicateResults(results: SkillSource[]): SkillSource[] {
|
||||
const seen = new Set<string>();
|
||||
return results.filter((source) => {
|
||||
const formatSuffix = source.sourceFormat && source.sourceFormat !== 'skill.md'
|
||||
? `::${source.sourceFormat}` : '';
|
||||
const key = `${source.owner}/${source.repo}/${source.path}${formatSuffix}`;
|
||||
if (seen.has(key)) {
|
||||
return false;
|
||||
}
|
||||
seen.add(key);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Check if error is a secondary/abuse rate limit (code search throttle)
|
||||
* These return 403 with "You have exceeded a secondary rate limit" or "abuse"
|
||||
*/
|
||||
private isSecondaryRateLimitError(error: unknown): boolean {
|
||||
if (!(error instanceof Error) || !('status' in error)) return false;
|
||||
const status = (error as { status: number }).status;
|
||||
return (
|
||||
status === 403 &&
|
||||
(error.message.includes('secondary rate limit') ||
|
||||
error.message.includes('abuse detection'))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract retry-after seconds from error response, default 60s
|
||||
*/
|
||||
private getRetryAfterSeconds(error: unknown): number {
|
||||
if (error instanceof Error && 'response' in error) {
|
||||
const resp = (error as { response?: { headers?: Record<string, string> } }).response;
|
||||
const retryAfter = resp?.headers?.['retry-after'];
|
||||
if (retryAfter) {
|
||||
return Math.max(parseInt(retryAfter, 10) || 60, 10);
|
||||
}
|
||||
}
|
||||
return 60;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if error is a primary rate limit error
|
||||
*/
|
||||
private isRateLimitError(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Error &&
|
||||
'status' in error &&
|
||||
(error as { status: number }).status === 403 &&
|
||||
error.message.includes('rate limit')
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if error is a not found error
|
||||
*/
|
||||
private isNotFoundError(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Error && 'status' in error && (error as { status: number }).status === 404
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if error is due to exceeding GitHub's 1000 result limit
|
||||
*/
|
||||
private isBeyondResultsLimit(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Error &&
|
||||
'status' in error &&
|
||||
(error as { status: number }).status === 422 &&
|
||||
error.message.includes('Cannot access beyond the first 1000 results')
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for rate limit to reset
|
||||
*/
|
||||
private async waitForRateLimit(): Promise<void> {
|
||||
console.log(' ⏳ Rate limited, waiting 60 seconds...');
|
||||
await new Promise((resolve) => setTimeout(resolve, 60000));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current GitHub API rate limit status (all tokens)
|
||||
*/
|
||||
async getRateLimitStatus(): Promise<{
|
||||
limit: number;
|
||||
remaining: number;
|
||||
resetAt: number;
|
||||
}> {
|
||||
const status = this.tokenManager.getStatus();
|
||||
return {
|
||||
limit: status.tokens[0]?.limit || 5000,
|
||||
remaining: status.globalRemaining,
|
||||
resetAt: status.nextReset,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if we have enough API budget to continue crawling.
|
||||
* Returns true if remaining quota is above the threshold percentage.
|
||||
* @param budgetPct - percentage of total limit to reserve (e.g., 0.33 = stop at 33% remaining)
|
||||
*/
|
||||
async checkBudget(budgetPct = 0.33): Promise<{ ok: boolean; remaining: number; limit: number }> {
|
||||
// Refresh all tokens to get accurate counts
|
||||
for (const tokenInfo of this.tokenManager.getStatus().tokens) {
|
||||
await this.tokenManager.refreshRateLimit(tokenInfo.token);
|
||||
}
|
||||
const status = this.tokenManager.getStatus();
|
||||
const totalLimit = status.tokens.reduce((sum, t) => sum + t.limit, 0);
|
||||
const remaining = status.globalRemaining;
|
||||
const threshold = Math.floor(totalLimit * budgetPct);
|
||||
return {
|
||||
ok: remaining > threshold,
|
||||
remaining,
|
||||
limit: totalLimit,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait until API budget is replenished above the threshold
|
||||
*/
|
||||
async waitForBudget(budgetPct = 0.33): Promise<void> {
|
||||
const status = this.tokenManager.getStatus();
|
||||
const nextReset = status.nextReset;
|
||||
const waitTime = Math.max(0, nextReset - Date.now()) + 2000;
|
||||
console.log(`\n⏳ API budget low. Waiting ${Math.ceil(waitTime / 1000)}s for rate limit reset...`);
|
||||
console.log(` Reset at: ${new Date(nextReset).toLocaleTimeString()}`);
|
||||
await new Promise((resolve) => setTimeout(resolve, waitTime));
|
||||
// Refresh after waiting
|
||||
for (const tokenInfo of this.tokenManager.getStatus().tokens) {
|
||||
await this.tokenManager.refreshRateLimit(tokenInfo.token);
|
||||
}
|
||||
const updated = await this.checkBudget(budgetPct);
|
||||
console.log(` Budget restored: ${updated.remaining}/${updated.limit} remaining\n`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new GitHubCrawler instance
|
||||
* @param tokenManager - Optional TokenManager instance (backward compatible: accepts token string and creates single-token manager)
|
||||
*/
|
||||
export function createCrawler(tokenManager?: TokenManager | string): GitHubCrawler {
|
||||
// Backward compatibility: if string is passed, create single-token manager
|
||||
if (typeof tokenManager === 'string') {
|
||||
return new GitHubCrawler(new TokenManager([tokenManager]));
|
||||
}
|
||||
return new GitHubCrawler(tokenManager);
|
||||
}
|
||||
190
services/indexer/src/direct-crawl.ts
Normal file
190
services/indexer/src/direct-crawl.ts
Normal file
@@ -0,0 +1,190 @@
|
||||
/**
|
||||
* Direct crawl script - bypasses the queue and runs crawl directly
|
||||
* Usage: npx tsup src/direct-crawl.ts --format cjs --outDir dist && node dist/direct-crawl.js
|
||||
*/
|
||||
|
||||
import pLimit from 'p-limit';
|
||||
import type { SkillSource, SourceFormat } from 'skillhub-core';
|
||||
import { createDb, skillQueries, categoryQueries, type Database } from '@skillhub/db';
|
||||
import { GitHubCrawler } from './crawler.js';
|
||||
import { SkillAnalyzer } from './analyzer.js';
|
||||
import { syncSkillToMeilisearch, logMeilisearchStatus } from './meilisearch-sync.js';
|
||||
|
||||
let db: Database | null = null;
|
||||
|
||||
function getDb(): Database {
|
||||
if (!db) {
|
||||
db = createDb(process.env.DATABASE_URL);
|
||||
}
|
||||
return db;
|
||||
}
|
||||
|
||||
/**
|
||||
* Index a single skill (copied from worker.ts pattern)
|
||||
*/
|
||||
async function indexSkill(
|
||||
crawler: GitHubCrawler,
|
||||
source: SkillSource,
|
||||
force = false
|
||||
): Promise<string | null> {
|
||||
const database = getDb();
|
||||
const analyzer = new SkillAnalyzer();
|
||||
|
||||
const sourceFormat: SourceFormat = source.sourceFormat || 'skill.md';
|
||||
|
||||
try {
|
||||
// Fetch skill content
|
||||
const content = await crawler.fetchSkillContent(source);
|
||||
|
||||
// Analyze the skill with format awareness
|
||||
const analysis = await analyzer.analyze(content, sourceFormat);
|
||||
|
||||
// Skip invalid skills
|
||||
if (!analysis.validation.isValid) {
|
||||
console.log(` ⚠️ Invalid: ${analysis.validation.errors.map((e) => e.message).join(', ')}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Generate skill ID with format suffix for non-SKILL.md
|
||||
const skillName = analysis.skill.metadata.name || source.path.split('/').pop() || 'skill';
|
||||
const formatSuffix = sourceFormat !== 'skill.md' ? `~${sourceFormat.replace('.', '')}` : '';
|
||||
const skillId = `${source.owner}/${source.repo}/${skillName}${formatSuffix}`;
|
||||
|
||||
// Check if skill is blocked (owner requested removal)
|
||||
const existing = await skillQueries.getById(database, skillId);
|
||||
if (existing?.isBlocked) {
|
||||
console.log(` 🚫 Blocked (owner requested removal)`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Check if we need to update (unless force)
|
||||
if (!force) {
|
||||
if (existing && existing.contentHash === analysis.meta.contentHash) {
|
||||
return null; // Unchanged, skip
|
||||
}
|
||||
}
|
||||
|
||||
// Upsert to database
|
||||
await skillQueries.upsert(database, {
|
||||
id: skillId,
|
||||
name: analysis.skill.metadata.name,
|
||||
description: analysis.skill.metadata.description,
|
||||
githubOwner: source.owner,
|
||||
githubRepo: source.repo,
|
||||
skillPath: source.path,
|
||||
branch: source.branch || content.repoMeta.defaultBranch,
|
||||
sourceFormat,
|
||||
version: analysis.skill.metadata.version,
|
||||
license: analysis.skill.metadata.license || content.repoMeta.license,
|
||||
author: analysis.skill.metadata.author,
|
||||
homepage: analysis.skill.metadata.homepage,
|
||||
compatibility: analysis.skill.metadata.compatibility,
|
||||
triggers: analysis.skill.metadata.triggers,
|
||||
githubStars: content.repoMeta.stars,
|
||||
githubForks: content.repoMeta.forks,
|
||||
securityScore: analysis.security.score,
|
||||
contentHash: analysis.meta.contentHash,
|
||||
rawContent: content.skillMd,
|
||||
indexedAt: new Date(),
|
||||
});
|
||||
|
||||
// Sync to Meilisearch
|
||||
await syncSkillToMeilisearch({
|
||||
id: skillId,
|
||||
name: analysis.skill.metadata.name,
|
||||
description: analysis.skill.metadata.description,
|
||||
githubOwner: source.owner,
|
||||
githubRepo: source.repo,
|
||||
compatibility: analysis.skill.metadata.compatibility,
|
||||
githubStars: content.repoMeta.stars,
|
||||
securityScore: analysis.security.score,
|
||||
indexedAt: new Date(),
|
||||
});
|
||||
|
||||
// Link to categories
|
||||
try {
|
||||
await categoryQueries.linkSkillToCategories(
|
||||
database,
|
||||
skillId,
|
||||
analysis.skill.metadata.name,
|
||||
analysis.skill.metadata.description || ''
|
||||
);
|
||||
} catch {
|
||||
// Category linking is optional
|
||||
}
|
||||
|
||||
console.log(` ✅ ${skillId} [${sourceFormat}] (⭐${content.repoMeta.stars})`);
|
||||
return skillId;
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'Unknown error';
|
||||
// Only log non-network errors
|
||||
if (!message.includes('socket') && !message.includes('network')) {
|
||||
console.error(` ❌ ${source.owner}/${source.repo}/${source.path} - ${message}`);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the full crawl directly
|
||||
*/
|
||||
async function runDirectCrawl() {
|
||||
console.log('🚀 Starting direct crawl (no queue)...\n');
|
||||
|
||||
const startTime = Date.now();
|
||||
const crawler = new GitHubCrawler();
|
||||
|
||||
// Check Meilisearch
|
||||
await logMeilisearchStatus();
|
||||
|
||||
// Discover all skill repositories
|
||||
console.log('\n📡 Discovering skill repositories...');
|
||||
const sources = await crawler.discoverSkillRepos({
|
||||
minStars: 0,
|
||||
maxPages: 50,
|
||||
});
|
||||
|
||||
console.log(`\n📊 Discovered ${sources.length} potential skills\n`);
|
||||
|
||||
// Index each skill with rate limiting
|
||||
const limit = pLimit(3);
|
||||
const results = { indexed: 0, failed: 0, skipped: 0 };
|
||||
|
||||
console.log('⏳ Indexing skills...\n');
|
||||
|
||||
const indexPromises = sources.map((source) =>
|
||||
limit(async () => {
|
||||
try {
|
||||
const skillId = await indexSkill(crawler, source, false);
|
||||
if (skillId) {
|
||||
results.indexed++;
|
||||
} else {
|
||||
results.skipped++;
|
||||
}
|
||||
} catch {
|
||||
results.failed++;
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
await Promise.all(indexPromises);
|
||||
|
||||
const duration = Math.round((Date.now() - startTime) / 1000);
|
||||
|
||||
console.log('\n' + '='.repeat(50));
|
||||
console.log('📊 Crawl Complete!');
|
||||
console.log('='.repeat(50));
|
||||
console.log(` ✅ Indexed: ${results.indexed}`);
|
||||
console.log(` ⏭️ Skipped: ${results.skipped}`);
|
||||
console.log(` ❌ Failed: ${results.failed}`);
|
||||
console.log(` ⏱️ Duration: ${duration}s`);
|
||||
console.log('='.repeat(50));
|
||||
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Run
|
||||
runDirectCrawl().catch((error) => {
|
||||
console.error('Fatal error:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
30
services/indexer/src/index.ts
Normal file
30
services/indexer/src/index.ts
Normal file
@@ -0,0 +1,30 @@
|
||||
// Crawler
|
||||
export { GitHubCrawler, createCrawler } from './crawler.js';
|
||||
export type {
|
||||
DiscoverOptions,
|
||||
SkillContent,
|
||||
FileInfo,
|
||||
ScriptFile,
|
||||
ReferenceFile,
|
||||
RepoMetadata,
|
||||
} from './crawler.js';
|
||||
|
||||
// Analyzer
|
||||
export { SkillAnalyzer, createAnalyzer } from './analyzer.js';
|
||||
export type { AnalysisResult, QualityScore, QualityFactor, AnalysisMeta } from './analyzer.js';
|
||||
|
||||
// Queue
|
||||
export {
|
||||
getQueue,
|
||||
getQueueEvents,
|
||||
scheduleFullCrawl,
|
||||
scheduleIncrementalCrawl,
|
||||
scheduleSkillIndex,
|
||||
setupRecurringJobs,
|
||||
getQueueStats,
|
||||
closeQueue,
|
||||
} from './queue.js';
|
||||
export type { IndexJobData, IndexJobResult } from './queue.js';
|
||||
|
||||
// Worker
|
||||
export { startWorker } from './worker.js';
|
||||
306
services/indexer/src/meilisearch-sync.ts
Normal file
306
services/indexer/src/meilisearch-sync.ts
Normal file
@@ -0,0 +1,306 @@
|
||||
import { MeiliSearch } from 'meilisearch';
|
||||
import type { MeiliSkillDocument } from '@skillhub/db';
|
||||
|
||||
/**
|
||||
* Meilisearch sync module for the indexer
|
||||
*
|
||||
* This module syncs skills to Meilisearch after they are indexed.
|
||||
* If MEILI_URL is not configured, sync operations are no-ops.
|
||||
*/
|
||||
|
||||
const SKILLS_INDEX = 'skills';
|
||||
|
||||
// Singleton client instance
|
||||
let client: MeiliSearch | null = null;
|
||||
let indexInitialized = false;
|
||||
|
||||
/**
|
||||
* Sanitize skill ID for Meilisearch
|
||||
* Meilisearch only allows alphanumeric, hyphens, and underscores in document IDs
|
||||
* We replace:
|
||||
* / with __ (slash to double underscore)
|
||||
* . with _dot_ (dot to _dot_)
|
||||
* Examples:
|
||||
* "anthropics/skills/pdf" -> "anthropics__skills__pdf"
|
||||
* "bdmorin/.claude/git" -> "bdmorin___dot_claude__git"
|
||||
* "user/repo.name/skill" -> "user__repo_dot_name__skill"
|
||||
*/
|
||||
function sanitizeIdForMeili(skillId: string): string {
|
||||
return skillId
|
||||
.replace(/\//g, '__') // slash -> double underscore
|
||||
.replace(/\./g, '_dot_'); // dot -> _dot_
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if Meilisearch is configured
|
||||
*/
|
||||
export function isMeilisearchConfigured(): boolean {
|
||||
return Boolean(process.env.MEILI_URL);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create the Meilisearch client
|
||||
*/
|
||||
function getMeilisearchClient(): MeiliSearch | null {
|
||||
if (!isMeilisearchConfigured()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!client) {
|
||||
client = new MeiliSearch({
|
||||
host: process.env.MEILI_URL!,
|
||||
apiKey: process.env.MEILI_MASTER_KEY,
|
||||
});
|
||||
}
|
||||
|
||||
return client;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the skills index with proper settings
|
||||
* This is called once when the first skill is synced
|
||||
*/
|
||||
async function initializeIndex(): Promise<void> {
|
||||
if (indexInitialized) return;
|
||||
|
||||
const meili = getMeilisearchClient();
|
||||
if (!meili) return;
|
||||
|
||||
try {
|
||||
// Create index if not exists
|
||||
try {
|
||||
await meili.getIndex(SKILLS_INDEX);
|
||||
} catch {
|
||||
console.log('Creating Meilisearch skills index...');
|
||||
await meili.createIndex(SKILLS_INDEX, { primaryKey: 'id' });
|
||||
}
|
||||
|
||||
const index = meili.index(SKILLS_INDEX);
|
||||
|
||||
// Configure searchable attributes (order matters for ranking)
|
||||
await index.updateSearchableAttributes([
|
||||
'name',
|
||||
'description',
|
||||
'githubOwner',
|
||||
'githubRepo',
|
||||
]);
|
||||
|
||||
// Configure filterable attributes
|
||||
await index.updateFilterableAttributes([
|
||||
'platforms',
|
||||
'isVerified',
|
||||
'isFeatured',
|
||||
'securityScore',
|
||||
'githubStars',
|
||||
]);
|
||||
|
||||
// Configure sortable attributes
|
||||
await index.updateSortableAttributes([
|
||||
'githubStars',
|
||||
'downloadCount',
|
||||
'rating',
|
||||
'indexedAt',
|
||||
]);
|
||||
|
||||
// Configure ranking rules
|
||||
await index.updateRankingRules([
|
||||
'words',
|
||||
'typo',
|
||||
'proximity',
|
||||
'attribute',
|
||||
'sort',
|
||||
'exactness',
|
||||
'githubStars:desc',
|
||||
]);
|
||||
|
||||
indexInitialized = true;
|
||||
console.log('Meilisearch skills index initialized');
|
||||
} catch (error) {
|
||||
console.error('Failed to initialize Meilisearch index:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync a single skill to Meilisearch
|
||||
*/
|
||||
export async function syncSkillToMeilisearch(skill: {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
githubOwner: string;
|
||||
githubRepo: string;
|
||||
compatibility?: { platforms?: string[] } | null;
|
||||
githubStars?: number | null;
|
||||
downloadCount?: number | null;
|
||||
rating?: number | null;
|
||||
ratingCount?: number | null;
|
||||
securityScore?: number | null;
|
||||
securityStatus?: 'pass' | 'warning' | 'fail' | null;
|
||||
isFeatured?: boolean | null;
|
||||
isVerified?: boolean | null;
|
||||
indexedAt?: Date | null;
|
||||
}): Promise<boolean> {
|
||||
const meili = getMeilisearchClient();
|
||||
if (!meili) {
|
||||
// Meilisearch not configured, skip silently
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
// Ensure index is initialized
|
||||
await initializeIndex();
|
||||
|
||||
const doc: MeiliSkillDocument = {
|
||||
id: sanitizeIdForMeili(skill.id),
|
||||
name: skill.name,
|
||||
description: skill.description,
|
||||
githubOwner: skill.githubOwner,
|
||||
githubRepo: skill.githubRepo,
|
||||
platforms: skill.compatibility?.platforms || [],
|
||||
githubStars: skill.githubStars || 0,
|
||||
downloadCount: skill.downloadCount || 0,
|
||||
rating: skill.rating || 0,
|
||||
ratingCount: skill.ratingCount || 0,
|
||||
securityScore: skill.securityScore || 0,
|
||||
securityStatus: skill.securityStatus || null,
|
||||
isFeatured: skill.isFeatured || false,
|
||||
isVerified: skill.isVerified || false,
|
||||
indexedAt: skill.indexedAt?.toISOString() || new Date().toISOString(),
|
||||
};
|
||||
|
||||
const index = meili.index<MeiliSkillDocument>(SKILLS_INDEX);
|
||||
await index.addDocuments([doc]);
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
// Log but don't fail - Meilisearch sync is optional
|
||||
console.error('Failed to sync skill to Meilisearch:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a skill from Meilisearch
|
||||
*/
|
||||
export async function deleteSkillFromMeilisearch(skillId: string): Promise<boolean> {
|
||||
const meili = getMeilisearchClient();
|
||||
if (!meili) return true;
|
||||
|
||||
try {
|
||||
const index = meili.index(SKILLS_INDEX);
|
||||
await index.deleteDocument(sanitizeIdForMeili(skillId));
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('Failed to delete skill from Meilisearch:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if Meilisearch is healthy
|
||||
*/
|
||||
export async function checkMeilisearchHealth(): Promise<boolean> {
|
||||
const meili = getMeilisearchClient();
|
||||
if (!meili) return false;
|
||||
|
||||
try {
|
||||
await meili.health();
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Log Meilisearch status on startup
|
||||
*/
|
||||
export async function logMeilisearchStatus(): Promise<void> {
|
||||
if (!isMeilisearchConfigured()) {
|
||||
console.log('Meilisearch: Not configured (MEILI_URL not set) - using PostgreSQL for search');
|
||||
return;
|
||||
}
|
||||
|
||||
const healthy = await checkMeilisearchHealth();
|
||||
if (healthy) {
|
||||
console.log(`Meilisearch: Connected to ${process.env.MEILI_URL}`);
|
||||
} else {
|
||||
console.warn(`Meilisearch: Unable to connect to ${process.env.MEILI_URL} - sync will be skipped`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync all skills from database to Meilisearch
|
||||
* This is useful for initial setup or recovering from sync issues
|
||||
*/
|
||||
export async function syncAllSkillsToMeilisearch(
|
||||
skills: Array<{
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
githubOwner: string;
|
||||
githubRepo: string;
|
||||
compatibility?: { platforms?: string[] } | null;
|
||||
githubStars?: number | null;
|
||||
downloadCount?: number | null;
|
||||
rating?: number | null;
|
||||
ratingCount?: number | null;
|
||||
securityScore?: number | null;
|
||||
securityStatus?: 'pass' | 'warning' | 'fail' | null;
|
||||
isFeatured?: boolean | null;
|
||||
isVerified?: boolean | null;
|
||||
indexedAt?: Date | null;
|
||||
}>
|
||||
): Promise<{ success: number; failed: number }> {
|
||||
const meili = getMeilisearchClient();
|
||||
if (!meili) {
|
||||
console.log('Meilisearch not configured, skipping bulk sync');
|
||||
return { success: 0, failed: 0 };
|
||||
}
|
||||
|
||||
// Ensure index is initialized
|
||||
await initializeIndex();
|
||||
|
||||
const results = { success: 0, failed: 0 };
|
||||
|
||||
// Convert skills to Meilisearch documents
|
||||
const documents: MeiliSkillDocument[] = skills.map((skill) => ({
|
||||
id: sanitizeIdForMeili(skill.id),
|
||||
name: skill.name,
|
||||
description: skill.description,
|
||||
githubOwner: skill.githubOwner,
|
||||
githubRepo: skill.githubRepo,
|
||||
platforms: skill.compatibility?.platforms || [],
|
||||
githubStars: skill.githubStars || 0,
|
||||
downloadCount: skill.downloadCount || 0,
|
||||
rating: skill.rating || 0,
|
||||
ratingCount: skill.ratingCount || 0,
|
||||
securityScore: skill.securityScore || 0,
|
||||
securityStatus: skill.securityStatus || null,
|
||||
isFeatured: skill.isFeatured || false,
|
||||
isVerified: skill.isVerified || false,
|
||||
indexedAt: skill.indexedAt?.toISOString() || new Date().toISOString(),
|
||||
}));
|
||||
|
||||
try {
|
||||
const index = meili.index<MeiliSkillDocument>(SKILLS_INDEX);
|
||||
|
||||
// Add documents in batches of 1000 for better performance
|
||||
const BATCH_SIZE = 1000;
|
||||
const totalBatches = Math.ceil(documents.length / BATCH_SIZE);
|
||||
|
||||
for (let i = 0; i < documents.length; i += BATCH_SIZE) {
|
||||
const batch = documents.slice(i, i + BATCH_SIZE);
|
||||
const batchNum = Math.floor(i / BATCH_SIZE) + 1;
|
||||
|
||||
await index.addDocuments(batch);
|
||||
results.success += batch.length;
|
||||
|
||||
console.log(`Synced batch ${batchNum}/${totalBatches} (${results.success}/${documents.length} skills) to Meilisearch`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Bulk sync to Meilisearch failed:', error);
|
||||
results.failed = documents.length - results.success;
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
44
services/indexer/src/octokit-pool.ts
Normal file
44
services/indexer/src/octokit-pool.ts
Normal file
@@ -0,0 +1,44 @@
|
||||
/**
|
||||
* Octokit Instance Pool
|
||||
* Caches Octokit instances per token to avoid recreation overhead
|
||||
*/
|
||||
|
||||
import { Octokit } from '@octokit/rest';
|
||||
import { TokenManager } from './token-manager.js';
|
||||
|
||||
export class OctokitPool {
|
||||
private instances: Map<string, Octokit> = new Map();
|
||||
private tokenManager: TokenManager;
|
||||
|
||||
constructor(tokenManager?: TokenManager) {
|
||||
this.tokenManager = tokenManager || TokenManager.getInstance();
|
||||
}
|
||||
|
||||
getInstance(token?: string): Octokit {
|
||||
const actualToken = token || this.tokenManager.getBestToken();
|
||||
|
||||
let instance = this.instances.get(actualToken);
|
||||
if (!instance) {
|
||||
instance = new Octokit({
|
||||
auth: actualToken,
|
||||
userAgent: 'SkillHub-Indexer/1.0',
|
||||
});
|
||||
this.instances.set(actualToken, instance);
|
||||
}
|
||||
|
||||
return instance;
|
||||
}
|
||||
|
||||
async getBestInstance(): Promise<Octokit> {
|
||||
const token = await this.tokenManager.checkAndRotate();
|
||||
return this.getInstance(token);
|
||||
}
|
||||
|
||||
updateStats(token: string, headers: Record<string, unknown>): void {
|
||||
this.tokenManager.updateTokenStats(token, headers);
|
||||
}
|
||||
|
||||
getTokenManager(): TokenManager {
|
||||
return this.tokenManager;
|
||||
}
|
||||
}
|
||||
311
services/indexer/src/queue.ts
Normal file
311
services/indexer/src/queue.ts
Normal file
@@ -0,0 +1,311 @@
|
||||
import { Queue, QueueEvents } from 'bullmq';
|
||||
import type { SkillSource } from 'skillhub-core';
|
||||
|
||||
const QUEUE_NAME = 'skill-indexing';
|
||||
|
||||
// Parse REDIS_URL if provided, otherwise use REDIS_HOST/PORT
|
||||
function getRedisConnection() {
|
||||
const redisUrl = process.env.REDIS_URL;
|
||||
if (redisUrl) {
|
||||
const url = new URL(redisUrl);
|
||||
const connection: {
|
||||
host: string;
|
||||
port: number;
|
||||
password?: string;
|
||||
username?: string;
|
||||
} = {
|
||||
host: url.hostname,
|
||||
port: parseInt(url.port || '6379'),
|
||||
};
|
||||
// Add authentication if present in URL
|
||||
if (url.password) {
|
||||
connection.password = decodeURIComponent(url.password);
|
||||
}
|
||||
if (url.username && url.username !== 'default') {
|
||||
connection.username = decodeURIComponent(url.username);
|
||||
}
|
||||
return connection;
|
||||
}
|
||||
return {
|
||||
host: process.env.REDIS_HOST || 'localhost',
|
||||
port: parseInt(process.env.REDIS_PORT || '6379'),
|
||||
};
|
||||
}
|
||||
|
||||
export interface IndexJobData {
|
||||
type:
|
||||
| 'full-crawl'
|
||||
| 'incremental'
|
||||
| 'single-skill'
|
||||
| 'index-skill'
|
||||
| 'discover-repos'
|
||||
| 'awesome-lists'
|
||||
| 'deep-scan'
|
||||
| 'full-enhanced'
|
||||
| 'process-add-requests'
|
||||
;
|
||||
source?: SkillSource;
|
||||
options?: {
|
||||
minStars?: number;
|
||||
updatedAfter?: string; // ISO date string
|
||||
force?: boolean;
|
||||
scanLimit?: number; // For deep-scan: max repos to scan
|
||||
};
|
||||
}
|
||||
|
||||
export interface IndexJobResult {
|
||||
success: boolean;
|
||||
skillId?: string;
|
||||
error?: string;
|
||||
stats?: {
|
||||
discovered?: number;
|
||||
indexed?: number;
|
||||
failed?: number;
|
||||
duration?: number;
|
||||
};
|
||||
}
|
||||
|
||||
let queue: Queue<IndexJobData, IndexJobResult> | null = null;
|
||||
let queueEvents: QueueEvents | null = null;
|
||||
|
||||
/**
|
||||
* Get or create the indexing queue
|
||||
*/
|
||||
export function getQueue(): Queue<IndexJobData, IndexJobResult> {
|
||||
if (!queue) {
|
||||
queue = new Queue<IndexJobData, IndexJobResult>(QUEUE_NAME, {
|
||||
connection: getRedisConnection(),
|
||||
defaultJobOptions: {
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: 'exponential',
|
||||
delay: 5000,
|
||||
},
|
||||
removeOnComplete: {
|
||||
age: 24 * 3600, // Keep completed jobs for 24 hours
|
||||
count: 1000,
|
||||
},
|
||||
removeOnFail: {
|
||||
age: 7 * 24 * 3600, // Keep failed jobs for 7 days
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
return queue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get queue events for monitoring
|
||||
*/
|
||||
export function getQueueEvents(): QueueEvents {
|
||||
if (!queueEvents) {
|
||||
queueEvents = new QueueEvents(QUEUE_NAME, {
|
||||
connection: getRedisConnection(),
|
||||
});
|
||||
}
|
||||
return queueEvents;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a full crawl job to discover all skills
|
||||
*/
|
||||
export async function scheduleFullCrawl(options?: IndexJobData['options']): Promise<string> {
|
||||
const q = getQueue();
|
||||
const job = await q.add(
|
||||
'full-crawl',
|
||||
{ type: 'full-crawl', options },
|
||||
{
|
||||
jobId: `full-crawl-${Date.now()}`,
|
||||
priority: 10,
|
||||
}
|
||||
);
|
||||
return job.id!;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add an incremental crawl job (only recent updates)
|
||||
*/
|
||||
export async function scheduleIncrementalCrawl(): Promise<string> {
|
||||
const q = getQueue();
|
||||
|
||||
// Look for skills updated in the last 24 hours
|
||||
const oneDayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000);
|
||||
|
||||
const job = await q.add(
|
||||
'incremental-crawl',
|
||||
{
|
||||
type: 'incremental',
|
||||
options: {
|
||||
updatedAfter: oneDayAgo.toISOString(),
|
||||
},
|
||||
},
|
||||
{
|
||||
jobId: `incremental-${Date.now()}`,
|
||||
priority: 5,
|
||||
}
|
||||
);
|
||||
return job.id!;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a single skill indexing job
|
||||
*/
|
||||
export async function scheduleSkillIndex(
|
||||
source: SkillSource,
|
||||
force = false
|
||||
): Promise<string> {
|
||||
const q = getQueue();
|
||||
const skillId = `${source.owner}/${source.repo}/${source.path}`;
|
||||
|
||||
const job = await q.add(
|
||||
'index-skill',
|
||||
{
|
||||
type: 'index-skill',
|
||||
source,
|
||||
options: { force },
|
||||
},
|
||||
{
|
||||
jobId: `skill-${skillId.replace(/\//g, '-')}-${Date.now()}`,
|
||||
priority: 1,
|
||||
}
|
||||
);
|
||||
return job.id!;
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup recurring jobs for production
|
||||
*
|
||||
* Schedule:
|
||||
* - Daily at 1:00 AM: Awesome lists discovery (fast, high-yield)
|
||||
* - Daily at 2:00 AM: Incremental crawl (existing code search)
|
||||
* - Daily at 3:00 AM: Deep scan of discovered repos
|
||||
* - Weekly Sunday at 4:00 AM: Full discovery (all strategies)
|
||||
* - Weekly Sunday at 5:00 AM: Full crawl (code search)
|
||||
*/
|
||||
export async function setupRecurringJobs(): Promise<void> {
|
||||
const q = getQueue();
|
||||
|
||||
// Remove existing repeatable jobs
|
||||
const repeatableJobs = await q.getRepeatableJobs();
|
||||
for (const job of repeatableJobs) {
|
||||
await q.removeRepeatableByKey(job.key);
|
||||
}
|
||||
|
||||
// Daily awesome lists discovery (1:00 AM) - fast, finds new repos quickly
|
||||
await q.add(
|
||||
'awesome-lists',
|
||||
{ type: 'awesome-lists' },
|
||||
{
|
||||
repeat: {
|
||||
pattern: '0 1 * * *', // Every day at 1 AM
|
||||
},
|
||||
jobId: 'recurring-awesome-lists',
|
||||
}
|
||||
);
|
||||
|
||||
// Daily incremental crawl (2:00 AM) - existing code search for updates
|
||||
await q.add(
|
||||
'incremental',
|
||||
{ type: 'incremental' },
|
||||
{
|
||||
repeat: {
|
||||
pattern: '0 2 * * *', // Every day at 2 AM
|
||||
},
|
||||
jobId: 'recurring-incremental',
|
||||
}
|
||||
);
|
||||
|
||||
// Daily deep scan (3:00 AM) - scan discovered repos for SKILL.md
|
||||
await q.add(
|
||||
'deep-scan',
|
||||
{ type: 'deep-scan', options: { scanLimit: 100 } },
|
||||
{
|
||||
repeat: {
|
||||
pattern: '0 3 * * *', // Every day at 3 AM
|
||||
},
|
||||
jobId: 'recurring-deep-scan',
|
||||
}
|
||||
);
|
||||
|
||||
|
||||
// Weekly full discovery (Sunday 5:00 AM) - all strategies
|
||||
await q.add(
|
||||
'discover-repos',
|
||||
{ type: 'discover-repos' },
|
||||
{
|
||||
repeat: {
|
||||
pattern: '0 5 * * 0', // Every Sunday at 5 AM
|
||||
},
|
||||
jobId: 'recurring-discover-repos',
|
||||
}
|
||||
);
|
||||
|
||||
// Weekly full crawl (Sunday 6:00 AM) - code search for all SKILL.md
|
||||
await q.add(
|
||||
'full-crawl',
|
||||
{ type: 'full-crawl' },
|
||||
{
|
||||
repeat: {
|
||||
pattern: '0 6 * * 0', // Every Sunday at 6 AM
|
||||
},
|
||||
jobId: 'recurring-full-crawl',
|
||||
}
|
||||
);
|
||||
|
||||
// Process add requests (every 6 hours) - index user-submitted skill requests
|
||||
await q.add(
|
||||
'process-add-requests',
|
||||
{ type: 'process-add-requests' },
|
||||
{
|
||||
repeat: {
|
||||
pattern: '30 */6 * * *', // Every 6 hours at :30
|
||||
},
|
||||
jobId: 'recurring-process-add-requests',
|
||||
}
|
||||
);
|
||||
|
||||
console.log('Recurring jobs scheduled:');
|
||||
console.log(' - Daily 1:00 AM: Awesome lists discovery');
|
||||
console.log(' - Daily 2:00 AM: Incremental crawl');
|
||||
console.log(' - Daily 3:00 AM: Deep scan (100 repos)');
|
||||
console.log(' - Sunday 5:00 AM: Full discovery');
|
||||
console.log(' - Sunday 6:00 AM: Full crawl');
|
||||
console.log(' - Every 6h: Process add requests');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get queue statistics
|
||||
*/
|
||||
export async function getQueueStats(): Promise<{
|
||||
waiting: number;
|
||||
active: number;
|
||||
completed: number;
|
||||
failed: number;
|
||||
delayed: number;
|
||||
}> {
|
||||
const q = getQueue();
|
||||
|
||||
const [waiting, active, completed, failed, delayed] = await Promise.all([
|
||||
q.getWaitingCount(),
|
||||
q.getActiveCount(),
|
||||
q.getCompletedCount(),
|
||||
q.getFailedCount(),
|
||||
q.getDelayedCount(),
|
||||
]);
|
||||
|
||||
return { waiting, active, completed, failed, delayed };
|
||||
}
|
||||
|
||||
/**
|
||||
* Close queue connections
|
||||
*/
|
||||
export async function closeQueue(): Promise<void> {
|
||||
if (queueEvents) {
|
||||
await queueEvents.close();
|
||||
queueEvents = null;
|
||||
}
|
||||
if (queue) {
|
||||
await queue.close();
|
||||
queue = null;
|
||||
}
|
||||
}
|
||||
119
services/indexer/src/skill-indexer.ts
Normal file
119
services/indexer/src/skill-indexer.ts
Normal file
@@ -0,0 +1,119 @@
|
||||
/**
|
||||
* Skill indexing utility
|
||||
* Shared between worker and CLI commands
|
||||
*/
|
||||
|
||||
import type { SourceFormat } from 'skillhub-core';
|
||||
import { createDb, skillQueries, categoryQueries, type Database } from '@skillhub/db';
|
||||
import type { GitHubCrawler } from './crawler.js';
|
||||
import { SkillAnalyzer } from './analyzer.js';
|
||||
import { syncSkillToMeilisearch } from './meilisearch-sync.js';
|
||||
|
||||
let db: Database | null = null;
|
||||
|
||||
function getDb(): Database {
|
||||
if (!db) {
|
||||
db = createDb(process.env.DATABASE_URL);
|
||||
}
|
||||
return db;
|
||||
}
|
||||
|
||||
/**
|
||||
* Index a single skill from a GitHub source
|
||||
*/
|
||||
export async function indexSkill(
|
||||
crawler: GitHubCrawler,
|
||||
source: { owner: string; repo: string; path: string; branch: string; sourceFormat?: SourceFormat },
|
||||
force = false
|
||||
): Promise<string | null> {
|
||||
const database = getDb();
|
||||
const analyzer = new SkillAnalyzer();
|
||||
const sourceFormat = source.sourceFormat || 'skill.md';
|
||||
|
||||
// Fetch skill content
|
||||
console.log(`Fetching ${source.owner}/${source.repo}/${source.path} [${sourceFormat}]...`);
|
||||
const content = await crawler.fetchSkillContent(source);
|
||||
|
||||
// Analyze the skill with format awareness
|
||||
const analysis = await analyzer.analyze(content, sourceFormat);
|
||||
|
||||
// Skip invalid skills
|
||||
if (!analysis.validation.isValid) {
|
||||
console.log(`Skipping invalid skill: ${analysis.validation.errors.map((e) => e.message).join(', ')}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Generate skill ID with format suffix for non-SKILL.md
|
||||
const skillName = analysis.skill.metadata.name || source.path.split('/').pop() || 'skill';
|
||||
const formatSuffix = sourceFormat !== 'skill.md' ? `~${sourceFormat.replace('.', '')}` : '';
|
||||
const skillId = `${source.owner}/${source.repo}/${skillName}${formatSuffix}`;
|
||||
|
||||
// Check if skill is blocked (owner requested removal)
|
||||
const existing = await skillQueries.getById(database, skillId);
|
||||
if (existing?.isBlocked) {
|
||||
console.log(`Skill ${skillId} is blocked, skipping`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Check if we need to update (unless force)
|
||||
if (!force) {
|
||||
if (existing && existing.contentHash === analysis.meta.contentHash) {
|
||||
console.log(`Skill ${skillId} unchanged, skipping`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Upsert to database
|
||||
await skillQueries.upsert(database, {
|
||||
id: skillId,
|
||||
name: analysis.skill.metadata.name,
|
||||
description: analysis.skill.metadata.description,
|
||||
githubOwner: source.owner,
|
||||
githubRepo: source.repo,
|
||||
skillPath: source.path,
|
||||
branch: source.branch || content.repoMeta.defaultBranch,
|
||||
sourceFormat,
|
||||
version: analysis.skill.metadata.version,
|
||||
license: analysis.skill.metadata.license || content.repoMeta.license,
|
||||
author: analysis.skill.metadata.author,
|
||||
homepage: analysis.skill.metadata.homepage,
|
||||
compatibility: analysis.skill.metadata.compatibility,
|
||||
triggers: analysis.skill.metadata.triggers,
|
||||
githubStars: content.repoMeta.stars,
|
||||
githubForks: content.repoMeta.forks,
|
||||
securityScore: analysis.security.score,
|
||||
contentHash: analysis.meta.contentHash,
|
||||
rawContent: content.skillMd,
|
||||
indexedAt: new Date(),
|
||||
});
|
||||
|
||||
// Sync to Meilisearch (optional - continues even if fails)
|
||||
await syncSkillToMeilisearch({
|
||||
id: skillId,
|
||||
name: analysis.skill.metadata.name,
|
||||
description: analysis.skill.metadata.description,
|
||||
githubOwner: source.owner,
|
||||
githubRepo: source.repo,
|
||||
compatibility: analysis.skill.metadata.compatibility,
|
||||
githubStars: content.repoMeta.stars,
|
||||
securityScore: analysis.security.score,
|
||||
indexedAt: new Date(),
|
||||
});
|
||||
|
||||
// Link skill to categories based on keywords
|
||||
try {
|
||||
const categories = await categoryQueries.linkSkillToCategories(
|
||||
database,
|
||||
skillId,
|
||||
analysis.skill.metadata.name,
|
||||
analysis.skill.metadata.description || ''
|
||||
);
|
||||
console.log(` -> Categories: [${categories.join(', ')}]`);
|
||||
} catch (error) {
|
||||
console.warn(` -> Failed to link categories:`, error);
|
||||
}
|
||||
|
||||
console.log(`Indexed: ${skillId} [${sourceFormat}] (security: ${analysis.security.score}, quality: ${analysis.quality.overall})`);
|
||||
|
||||
return skillId;
|
||||
}
|
||||
203
services/indexer/src/strategies/awesome-list.ts
Normal file
203
services/indexer/src/strategies/awesome-list.ts
Normal file
@@ -0,0 +1,203 @@
|
||||
import type { Octokit } from '@octokit/rest';
|
||||
import { TokenManager } from '../token-manager.js';
|
||||
import { OctokitPool } from '../octokit-pool.js';
|
||||
|
||||
export interface RepoReference {
|
||||
owner: string;
|
||||
repo: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Known awesome lists that contain skill repositories
|
||||
*/
|
||||
const KNOWN_AWESOME_LISTS = [
|
||||
{ owner: 'travisvn', repo: 'awesome-claude-skills', readme: 'README.md' },
|
||||
{ owner: 'VoltAgent', repo: 'awesome-claude-skills', readme: 'README.md' },
|
||||
{ owner: 'ComposioHQ', repo: 'awesome-claude-skills', readme: 'README.md' },
|
||||
{ owner: 'skillmatic-ai', repo: 'awesome-agent-skills', readme: 'README.md' },
|
||||
{ owner: 'github', repo: 'awesome-copilot', readme: 'README.md' },
|
||||
];
|
||||
|
||||
/**
|
||||
* Awesome List Crawler Strategy
|
||||
* Parses README files from curated "awesome" lists to discover skill repositories
|
||||
*/
|
||||
export class AwesomeListCrawler {
|
||||
private octokitPool: OctokitPool;
|
||||
private tokenManager: TokenManager;
|
||||
|
||||
constructor(tokenManager?: TokenManager) {
|
||||
this.tokenManager = tokenManager || TokenManager.getInstance();
|
||||
this.octokitPool = new OctokitPool(this.tokenManager);
|
||||
}
|
||||
|
||||
private async getOctokit(): Promise<Octokit> {
|
||||
return this.octokitPool.getBestInstance();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all known awesome lists
|
||||
*/
|
||||
getKnownLists() {
|
||||
return KNOWN_AWESOME_LISTS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Crawl all known awesome lists
|
||||
*/
|
||||
async crawlAllLists(): Promise<Map<string, RepoReference[]>> {
|
||||
const results = new Map<string, RepoReference[]>();
|
||||
|
||||
for (const list of KNOWN_AWESOME_LISTS) {
|
||||
try {
|
||||
console.log(`Parsing awesome list: ${list.owner}/${list.repo}`);
|
||||
const repos = await this.parseAwesomeList(list.owner, list.repo, list.readme);
|
||||
results.set(`${list.owner}/${list.repo}`, repos);
|
||||
console.log(` Found ${repos.length} repository references`);
|
||||
} catch (error) {
|
||||
console.warn(`Failed to parse ${list.owner}/${list.repo}:`, error);
|
||||
results.set(`${list.owner}/${list.repo}`, []);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an awesome list README to extract GitHub repository references
|
||||
*/
|
||||
async parseAwesomeList(
|
||||
owner: string,
|
||||
repo: string,
|
||||
readmePath = 'README.md'
|
||||
): Promise<RepoReference[]> {
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const response = await octokit.repos.getContent({
|
||||
owner,
|
||||
repo,
|
||||
path: readmePath,
|
||||
});
|
||||
|
||||
if (!('content' in response.data)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const content = Buffer.from(response.data.content, 'base64').toString('utf-8');
|
||||
return this.extractGitHubRepos(content);
|
||||
} catch (error) {
|
||||
console.warn(`Failed to fetch ${owner}/${repo}/${readmePath}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract GitHub repository references from markdown content
|
||||
*/
|
||||
extractGitHubRepos(content: string): RepoReference[] {
|
||||
const repos: RepoReference[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
// Pattern to match GitHub URLs: github.com/owner/repo
|
||||
// Supports various formats:
|
||||
// - https://github.com/owner/repo
|
||||
// - http://github.com/owner/repo
|
||||
// - github.com/owner/repo
|
||||
// - [text](https://github.com/owner/repo)
|
||||
const patterns = [
|
||||
/https?:\/\/github\.com\/([a-zA-Z0-9_-]+)\/([a-zA-Z0-9_.-]+)/g,
|
||||
/github\.com\/([a-zA-Z0-9_-]+)\/([a-zA-Z0-9_.-]+)/g,
|
||||
];
|
||||
|
||||
for (const pattern of patterns) {
|
||||
let match;
|
||||
while ((match = pattern.exec(content)) !== null) {
|
||||
const owner = match[1];
|
||||
let repo = match[2];
|
||||
|
||||
// Clean up repo name (remove trailing slashes, .git, anchors, etc.)
|
||||
repo = repo.replace(/\/$/, '').replace(/\.git$/, '').split('/')[0].split('#')[0];
|
||||
|
||||
// Skip invalid repos
|
||||
if (!owner || !repo || repo.length === 0) continue;
|
||||
|
||||
// Skip common non-repo patterns
|
||||
if (
|
||||
repo === 'issues' ||
|
||||
repo === 'pulls' ||
|
||||
repo === 'blob' ||
|
||||
repo === 'tree' ||
|
||||
repo === 'raw' ||
|
||||
repo === 'releases'
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const key = `${owner}/${repo}`.toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
repos.push({
|
||||
owner,
|
||||
repo,
|
||||
url: `https://github.com/${owner}/${repo}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return repos;
|
||||
}
|
||||
|
||||
/**
|
||||
* Discover additional awesome lists from GitHub search
|
||||
*/
|
||||
async discoverAwesomeLists(): Promise<RepoReference[]> {
|
||||
const searchQueries = [
|
||||
'awesome-claude-skills in:name',
|
||||
'awesome-agent-skills in:name',
|
||||
'awesome claude skills in:description',
|
||||
'awesome ai skills in:description',
|
||||
];
|
||||
|
||||
const allRepos: RepoReference[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const query of searchQueries) {
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const response = await octokit.search.repos({
|
||||
q: query,
|
||||
sort: 'stars',
|
||||
order: 'desc',
|
||||
per_page: 30,
|
||||
});
|
||||
|
||||
for (const repo of response.data.items) {
|
||||
if (!repo.owner) continue;
|
||||
const key = `${repo.owner.login}/${repo.name}`.toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
allRepos.push({
|
||||
owner: repo.owner.login,
|
||||
repo: repo.name,
|
||||
url: repo.html_url,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`Search query failed: ${query}`, error);
|
||||
}
|
||||
}
|
||||
|
||||
return allRepos;
|
||||
}
|
||||
}
|
||||
|
||||
export function createAwesomeListCrawler(tokenManager?: TokenManager | string): AwesomeListCrawler {
|
||||
// Backward compatibility
|
||||
if (typeof tokenManager === 'string') {
|
||||
return new AwesomeListCrawler(new TokenManager([tokenManager]));
|
||||
}
|
||||
return new AwesomeListCrawler(tokenManager);
|
||||
}
|
||||
261
services/indexer/src/strategies/deep-scan.ts
Normal file
261
services/indexer/src/strategies/deep-scan.ts
Normal file
@@ -0,0 +1,261 @@
|
||||
import type { Octokit } from '@octokit/rest';
|
||||
import { TokenManager } from '../token-manager.js';import { OctokitPool } from '../octokit-pool.js';
|
||||
import type { SkillSource } from 'skillhub-core';
|
||||
import { INSTRUCTION_FILE_PATTERNS, type SourceFormat } from 'skillhub-core';
|
||||
|
||||
/**
|
||||
* Deep Scan Strategy
|
||||
* Uses Git Trees API to recursively scan entire repositories for SKILL.md files
|
||||
* Can discover skills that aren't found by code search
|
||||
*/
|
||||
export class DeepScanCrawler {
|
||||
private octokitPool: OctokitPool;
|
||||
private tokenManager: TokenManager;
|
||||
|
||||
constructor(tokenManager?: TokenManager) {
|
||||
this.tokenManager = tokenManager || TokenManager.getInstance();
|
||||
this.octokitPool = new OctokitPool(this.tokenManager);
|
||||
}
|
||||
|
||||
private async getOctokit(): Promise<Octokit> {
|
||||
return this.octokitPool.getBestInstance();
|
||||
}
|
||||
|
||||
private getCurrentToken(): string {
|
||||
return this.tokenManager.getBestToken();
|
||||
}
|
||||
|
||||
/**
|
||||
* Deep scan a repository for all SKILL.md files using Git Trees API
|
||||
* This is more thorough than code search as it scans the entire repo
|
||||
*/
|
||||
async scanRepository(owner: string, repo: string): Promise<SkillSource[]> {
|
||||
const skills: SkillSource[] = [];
|
||||
|
||||
try {
|
||||
// Get repository info for default branch
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const repoInfo = await octokit.repos.get({ owner, repo });
|
||||
this.octokitPool.updateStats(token, repoInfo.headers);
|
||||
|
||||
if (repoInfo.data.archived) {
|
||||
console.log(` Skipping archived repo: ${owner}/${repo}`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const defaultBranch = repoInfo.data.default_branch;
|
||||
|
||||
// Get the full tree recursively
|
||||
const tree = await octokit.git.getTree({
|
||||
owner,
|
||||
repo,
|
||||
tree_sha: defaultBranch,
|
||||
recursive: 'true',
|
||||
});
|
||||
this.octokitPool.updateStats(token, tree.headers);
|
||||
|
||||
// Find all instruction files (SKILL.md, .cursorrules, .windsurfrules, AGENTS.md, etc.)
|
||||
const instructionFiles = tree.data.tree.filter(
|
||||
(item) => {
|
||||
if (item.type !== 'blob' || !item.path) return false;
|
||||
return INSTRUCTION_FILE_PATTERNS.some(pattern => {
|
||||
if (pattern.rootOnly) return item.path === pattern.filename;
|
||||
if (pattern.pathFilter) return item.path!.includes(pattern.pathFilter) && (item.path!.endsWith('/' + pattern.filename) || item.path === pattern.filename);
|
||||
return item.path!.endsWith('/' + pattern.filename) || item.path === pattern.filename;
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
for (const file of instructionFiles) {
|
||||
if (!file.path) continue;
|
||||
const matchedPattern = INSTRUCTION_FILE_PATTERNS.find(pattern => {
|
||||
if (pattern.rootOnly) return file.path === pattern.filename;
|
||||
if (pattern.pathFilter) return file.path!.includes(pattern.pathFilter) && (file.path!.endsWith('/' + pattern.filename) || file.path === pattern.filename);
|
||||
return file.path!.endsWith('/' + pattern.filename) || file.path === pattern.filename;
|
||||
});
|
||||
if (!matchedPattern) continue;
|
||||
|
||||
const escapedFilename = matchedPattern.filename.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const skillPath = file.path.replace(new RegExp(`/?${escapedFilename}$`), '') || '.';
|
||||
|
||||
skills.push({
|
||||
owner,
|
||||
repo,
|
||||
path: skillPath,
|
||||
branch: defaultBranch,
|
||||
sourceFormat: matchedPattern.format,
|
||||
});
|
||||
}
|
||||
if (skills.length > 0) {
|
||||
console.log(` Found ${skills.length} skills in ${owner}/${repo}`);
|
||||
}
|
||||
|
||||
return skills;
|
||||
} catch (error) {
|
||||
if (this.isNotFoundError(error)) {
|
||||
return [];
|
||||
}
|
||||
if (this.isTruncatedError(error)) {
|
||||
// Tree is truncated (>100k files), fall back to directory listing
|
||||
console.log(` Repository ${owner}/${repo} is too large, using fallback scan`);
|
||||
return this.fallbackScan(owner, repo);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fallback scan for large repositories - scan known skill directories
|
||||
*/
|
||||
private async fallbackScan(owner: string, repo: string): Promise<SkillSource[]> {
|
||||
const skills: SkillSource[] = [];
|
||||
const knownPaths = ['skills', '.claude/skills', '.github/skills', '.codex/skills', ''];
|
||||
|
||||
for (const basePath of knownPaths) {
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
|
||||
const response = await octokit.repos.getContent({
|
||||
owner,
|
||||
repo,
|
||||
path: basePath || '.',
|
||||
});
|
||||
|
||||
this.octokitPool.updateStats(token, response.headers);
|
||||
|
||||
if (!Array.isArray(response.data)) continue;
|
||||
|
||||
// Check for SKILL.md at this level
|
||||
const hasSkillMd = response.data.some((item) => item.name === 'SKILL.md');
|
||||
if (hasSkillMd) {
|
||||
skills.push({
|
||||
owner,
|
||||
repo,
|
||||
path: basePath || '.',
|
||||
branch: 'main',
|
||||
sourceFormat: 'skill.md' as SourceFormat,
|
||||
});
|
||||
}
|
||||
|
||||
// Check subdirectories for SKILL.md
|
||||
const dirs = response.data.filter((item) => item.type === 'dir');
|
||||
for (const dir of dirs) {
|
||||
try {
|
||||
const subOctokit = await this.getOctokit();
|
||||
const subToken = this.getCurrentToken();
|
||||
const subDir = await subOctokit.repos.getContent({
|
||||
owner,
|
||||
repo,
|
||||
path: dir.path,
|
||||
});
|
||||
this.octokitPool.updateStats(subToken, subDir.headers);
|
||||
|
||||
if (Array.isArray(subDir.data)) {
|
||||
const hasSubSkillMd = subDir.data.some((item) => item.name === 'SKILL.md');
|
||||
if (hasSubSkillMd) {
|
||||
skills.push({
|
||||
owner,
|
||||
repo,
|
||||
path: dir.path,
|
||||
branch: 'main',
|
||||
sourceFormat: 'skill.md' as SourceFormat,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Skip inaccessible directories
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Path doesn't exist, continue
|
||||
}
|
||||
}
|
||||
|
||||
// Check for root-level instruction files (.cursorrules, .windsurfrules, AGENTS.md)
|
||||
const rootFiles = INSTRUCTION_FILE_PATTERNS.filter(p => p.format !== 'skill.md');
|
||||
for (const pattern of rootFiles) {
|
||||
try {
|
||||
const filePath = pattern.pathFilter ? `${pattern.pathFilter}${pattern.filename}` : pattern.filename;
|
||||
const fileContent = await this.getFileContent(owner, repo, filePath);
|
||||
if (fileContent && fileContent.length >= pattern.minContentLength) {
|
||||
skills.push({
|
||||
owner,
|
||||
repo,
|
||||
path: '.',
|
||||
branch: 'main',
|
||||
sourceFormat: pattern.format,
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// File doesn't exist
|
||||
}
|
||||
}
|
||||
|
||||
return skills;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan multiple repositories
|
||||
*/
|
||||
async scanRepositories(
|
||||
repos: Array<{ owner: string; repo: string }>
|
||||
): Promise<Map<string, SkillSource[]>> {
|
||||
const results = new Map<string, SkillSource[]>();
|
||||
|
||||
for (const { owner, repo } of repos) {
|
||||
try {
|
||||
console.log(`Deep scanning: ${owner}/${repo}`);
|
||||
const skills = await this.scanRepository(owner, repo);
|
||||
results.set(`${owner}/${repo}`, skills);
|
||||
} catch (error) {
|
||||
console.warn(` Failed to scan ${owner}/${repo}:`, error);
|
||||
results.set(`${owner}/${repo}`, []);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get file content from a repository
|
||||
*/
|
||||
async getFileContent(owner: string, repo: string, path: string): Promise<string | null> {
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const response = await octokit.repos.getContent({
|
||||
owner,
|
||||
repo,
|
||||
path,
|
||||
});
|
||||
this.octokitPool.updateStats(token, response.headers);
|
||||
|
||||
if ('content' in response.data && response.data.type === 'file') {
|
||||
return Buffer.from(response.data.content, 'base64').toString('utf-8');
|
||||
}
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private isNotFoundError(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Error && 'status' in error && (error as { status: number }).status === 404
|
||||
);
|
||||
}
|
||||
|
||||
private isTruncatedError(error: unknown): boolean {
|
||||
// Git Trees API returns truncated: true for repos >100k files
|
||||
return error instanceof Error && error.message.includes('truncated');
|
||||
}
|
||||
}
|
||||
|
||||
export function createDeepScanCrawler(tokenManager?: TokenManager | string): DeepScanCrawler {
|
||||
if (typeof tokenManager === 'string') {
|
||||
return new DeepScanCrawler(new TokenManager([tokenManager]));
|
||||
}
|
||||
return new DeepScanCrawler(tokenManager);
|
||||
}
|
||||
194
services/indexer/src/strategies/fork-network.ts
Normal file
194
services/indexer/src/strategies/fork-network.ts
Normal file
@@ -0,0 +1,194 @@
|
||||
import type { Octokit } from '@octokit/rest';
|
||||
import { TokenManager } from '../token-manager.js';import { OctokitPool } from '../octokit-pool.js';
|
||||
|
||||
export interface ForkInfo {
|
||||
owner: string;
|
||||
repo: string;
|
||||
stars: number;
|
||||
updatedAt: string;
|
||||
isArchived: boolean;
|
||||
defaultBranch: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fork Network Strategy
|
||||
* Discovers skills by traversing fork networks of known skill repositories
|
||||
* Forks often contain modified or additional skills
|
||||
*/
|
||||
export class ForkNetworkCrawler {
|
||||
private octokitPool: OctokitPool;
|
||||
private tokenManager: TokenManager;
|
||||
|
||||
constructor(tokenManager?: TokenManager) {
|
||||
this.tokenManager = tokenManager || TokenManager.getInstance();
|
||||
this.octokitPool = new OctokitPool(this.tokenManager);
|
||||
}
|
||||
|
||||
private async getOctokit(): Promise<Octokit> {
|
||||
return this.octokitPool.getBestInstance();
|
||||
}
|
||||
|
||||
private getCurrentToken(): string {
|
||||
return this.tokenManager.getBestToken();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all forks of a repository
|
||||
*/
|
||||
async getForks(owner: string, repo: string, maxPages = 10): Promise<ForkInfo[]> {
|
||||
const forks: ForkInfo[] = [];
|
||||
let page = 1;
|
||||
|
||||
while (page <= maxPages) {
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
|
||||
const response = await octokit.repos.listForks({
|
||||
owner,
|
||||
repo,
|
||||
sort: 'stargazers',
|
||||
per_page: 100,
|
||||
page,
|
||||
});
|
||||
|
||||
this.octokitPool.updateStats(token, response.headers);
|
||||
|
||||
if (response.data.length === 0) break;
|
||||
|
||||
for (const fork of response.data) {
|
||||
// Skip old/abandoned forks (not updated in 1 year)
|
||||
const updatedAt = fork.updated_at ? new Date(fork.updated_at) : new Date(0);
|
||||
const oneYearAgo = new Date();
|
||||
oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1);
|
||||
|
||||
if (updatedAt < oneYearAgo) continue;
|
||||
|
||||
forks.push({
|
||||
owner: fork.owner.login,
|
||||
repo: fork.name,
|
||||
stars: fork.stargazers_count ?? 0,
|
||||
updatedAt: fork.updated_at ?? new Date().toISOString(),
|
||||
isArchived: fork.archived ?? false,
|
||||
defaultBranch: fork.default_branch ?? 'main',
|
||||
});
|
||||
}
|
||||
|
||||
if (response.data.length < 100) break;
|
||||
page++;
|
||||
} catch (error) {
|
||||
console.warn(`Failed to get forks page ${page} for ${owner}/${repo}:`, error);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return forks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get forks for multiple seed repositories
|
||||
*/
|
||||
async discoverFromSeedRepos(
|
||||
seedRepos: Array<{ owner: string; repo: string }>
|
||||
): Promise<ForkInfo[]> {
|
||||
const allForks: ForkInfo[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const seed of seedRepos) {
|
||||
try {
|
||||
console.log(`Fetching forks of ${seed.owner}/${seed.repo}...`);
|
||||
const forks = await this.getForks(seed.owner, seed.repo);
|
||||
|
||||
for (const fork of forks) {
|
||||
const key = `${fork.owner}/${fork.repo}`.toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
allForks.push(fork);
|
||||
}
|
||||
|
||||
console.log(` Found ${forks.length} active forks`);
|
||||
} catch (error) {
|
||||
console.warn(`Failed to get forks for ${seed.owner}/${seed.repo}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Total unique forks discovered: ${allForks.length}`);
|
||||
return allForks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get parent/source repository (for finding the original if we have a fork)
|
||||
*/
|
||||
async getParentRepo(owner: string, repo: string): Promise<{ owner: string; repo: string } | null> {
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const response = await octokit.repos.get({ owner, repo });
|
||||
this.octokitPool.updateStats(token, response.headers);
|
||||
|
||||
if (response.data.fork && response.data.parent) {
|
||||
return {
|
||||
owner: response.data.parent.owner.login,
|
||||
repo: response.data.parent.name,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all repositories in the same network (parent + siblings)
|
||||
*/
|
||||
async getNetworkRepos(owner: string, repo: string): Promise<ForkInfo[]> {
|
||||
const repos: ForkInfo[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
// Get parent if this is a fork
|
||||
const parent = await this.getParentRepo(owner, repo);
|
||||
const rootOwner = parent?.owner || owner;
|
||||
const rootRepo = parent?.repo || repo;
|
||||
|
||||
// Add root repo
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const rootInfo = await octokit.repos.get({ owner: rootOwner, repo: rootRepo });
|
||||
this.octokitPool.updateStats(token, rootInfo.headers);
|
||||
|
||||
const rootKey = `${rootOwner}/${rootRepo}`.toLowerCase();
|
||||
seen.add(rootKey);
|
||||
|
||||
repos.push({
|
||||
owner: rootOwner,
|
||||
repo: rootRepo,
|
||||
stars: rootInfo.data.stargazers_count,
|
||||
updatedAt: rootInfo.data.updated_at,
|
||||
isArchived: rootInfo.data.archived,
|
||||
defaultBranch: rootInfo.data.default_branch,
|
||||
});
|
||||
} catch {
|
||||
// Root not accessible
|
||||
}
|
||||
|
||||
// Get all forks of root
|
||||
const forks = await this.getForks(rootOwner, rootRepo);
|
||||
for (const fork of forks) {
|
||||
const key = `${fork.owner}/${fork.repo}`.toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
repos.push(fork);
|
||||
}
|
||||
|
||||
return repos;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export function createForkNetworkCrawler(tokenManager?: TokenManager | string): ForkNetworkCrawler {
|
||||
if (typeof tokenManager === 'string') {
|
||||
return new ForkNetworkCrawler(new TokenManager([tokenManager]));
|
||||
}
|
||||
return new ForkNetworkCrawler(tokenManager);
|
||||
}
|
||||
292
services/indexer/src/strategies/index.ts
Normal file
292
services/indexer/src/strategies/index.ts
Normal file
@@ -0,0 +1,292 @@
|
||||
/**
|
||||
* Strategy Orchestrator
|
||||
* Coordinates multiple discovery strategies for comprehensive skill discovery
|
||||
*/
|
||||
|
||||
import { createAwesomeListCrawler } from './awesome-list.js';
|
||||
import { createTopicSearchCrawler } from './topic-search.js';
|
||||
import { createDeepScanCrawler } from './deep-scan.js';
|
||||
import { createForkNetworkCrawler } from './fork-network.js';
|
||||
import { TokenManager } from '../token-manager.js';
|
||||
import type { SkillSource } from 'skillhub-core';
|
||||
|
||||
export interface DiscoveryResult {
|
||||
source: string;
|
||||
repos: Array<{
|
||||
owner: string;
|
||||
repo: string;
|
||||
stars?: number;
|
||||
discoveredVia: string;
|
||||
}>;
|
||||
skills: SkillSource[];
|
||||
}
|
||||
|
||||
export interface DiscoveryStats {
|
||||
totalReposDiscovered: number;
|
||||
totalSkillsFound: number;
|
||||
byStrategy: {
|
||||
[key: string]: {
|
||||
repos: number;
|
||||
skills: number;
|
||||
};
|
||||
};
|
||||
duration: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strategy Orchestrator - coordinates all discovery strategies
|
||||
*/
|
||||
export class StrategyOrchestrator {
|
||||
private awesomeCrawler: ReturnType<typeof createAwesomeListCrawler>;
|
||||
private topicCrawler: ReturnType<typeof createTopicSearchCrawler>;
|
||||
private deepScanCrawler: ReturnType<typeof createDeepScanCrawler>;
|
||||
private forkCrawler: ReturnType<typeof createForkNetworkCrawler>;
|
||||
private tokenManager: TokenManager;
|
||||
|
||||
constructor(tokenManager?: TokenManager) {
|
||||
this.tokenManager = tokenManager || TokenManager.getInstance();
|
||||
this.awesomeCrawler = createAwesomeListCrawler(this.tokenManager);
|
||||
this.topicCrawler = createTopicSearchCrawler(this.tokenManager);
|
||||
this.deepScanCrawler = createDeepScanCrawler(this.tokenManager);
|
||||
this.forkCrawler = createForkNetworkCrawler(this.tokenManager);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run awesome list discovery
|
||||
*/
|
||||
async runAwesomeListStrategy(): Promise<DiscoveryResult> {
|
||||
console.log('\n=== Running Awesome List Strategy ===');
|
||||
const startTime = Date.now();
|
||||
|
||||
const listResults = await this.awesomeCrawler.crawlAllLists();
|
||||
|
||||
const repos: DiscoveryResult['repos'] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const repoRefs of listResults.values()) {
|
||||
for (const ref of repoRefs) {
|
||||
const key = `${ref.owner}/${ref.repo}`.toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
repos.push({
|
||||
owner: ref.owner,
|
||||
repo: ref.repo,
|
||||
discoveredVia: 'awesome-list',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Awesome list strategy completed in ${Date.now() - startTime}ms`);
|
||||
console.log(`Discovered ${repos.length} repositories from ${listResults.size} lists`);
|
||||
|
||||
return {
|
||||
source: 'awesome-list',
|
||||
repos,
|
||||
skills: [],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run topic search discovery
|
||||
*/
|
||||
async runTopicSearchStrategy(): Promise<DiscoveryResult> {
|
||||
console.log('\n=== Running Topic Search Strategy ===');
|
||||
const startTime = Date.now();
|
||||
|
||||
const repoResults = await this.topicCrawler.discoverAll();
|
||||
|
||||
const repos = repoResults.map((r) => ({
|
||||
owner: r.owner,
|
||||
repo: r.repo,
|
||||
stars: r.stars,
|
||||
discoveredVia: 'topic-search',
|
||||
}));
|
||||
|
||||
console.log(`Topic search strategy completed in ${Date.now() - startTime}ms`);
|
||||
console.log(`Discovered ${repos.length} repositories`);
|
||||
|
||||
return {
|
||||
source: 'topic-search',
|
||||
repos,
|
||||
skills: [],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run deep scan on discovered repositories
|
||||
*/
|
||||
async runDeepScanStrategy(
|
||||
repos: Array<{ owner: string; repo: string }>
|
||||
): Promise<DiscoveryResult> {
|
||||
console.log('\n=== Running Deep Scan Strategy ===');
|
||||
const startTime = Date.now();
|
||||
|
||||
const scanResults = await this.deepScanCrawler.scanRepositories(repos);
|
||||
|
||||
const allSkills: SkillSource[] = [];
|
||||
for (const skills of scanResults.values()) {
|
||||
allSkills.push(...skills);
|
||||
}
|
||||
|
||||
console.log(`Deep scan strategy completed in ${Date.now() - startTime}ms`);
|
||||
console.log(`Found ${allSkills.length} skills in ${repos.length} repositories`);
|
||||
|
||||
return {
|
||||
source: 'deep-scan',
|
||||
repos: [],
|
||||
skills: allSkills,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run fork network discovery
|
||||
*/
|
||||
async runForkNetworkStrategy(
|
||||
seedRepos: Array<{ owner: string; repo: string }>
|
||||
): Promise<DiscoveryResult> {
|
||||
console.log('\n=== Running Fork Network Strategy ===');
|
||||
const startTime = Date.now();
|
||||
|
||||
const forks = await this.forkCrawler.discoverFromSeedRepos(seedRepos);
|
||||
|
||||
const repos = forks
|
||||
.filter((f) => !f.isArchived)
|
||||
.map((f) => ({
|
||||
owner: f.owner,
|
||||
repo: f.repo,
|
||||
stars: f.stars,
|
||||
discoveredVia: 'fork-network',
|
||||
}));
|
||||
|
||||
console.log(`Fork network strategy completed in ${Date.now() - startTime}ms`);
|
||||
console.log(`Discovered ${repos.length} active forks`);
|
||||
|
||||
return {
|
||||
source: 'fork-network',
|
||||
repos,
|
||||
skills: [],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run all strategies and return combined results
|
||||
*/
|
||||
async runAllStrategies(): Promise<{
|
||||
repos: Array<{
|
||||
owner: string;
|
||||
repo: string;
|
||||
stars?: number;
|
||||
discoveredVia: string;
|
||||
}>;
|
||||
stats: DiscoveryStats;
|
||||
}> {
|
||||
const startTime = Date.now();
|
||||
const stats: DiscoveryStats = {
|
||||
totalReposDiscovered: 0,
|
||||
totalSkillsFound: 0,
|
||||
byStrategy: {},
|
||||
duration: 0,
|
||||
};
|
||||
|
||||
const allRepos: Map<string, {
|
||||
owner: string;
|
||||
repo: string;
|
||||
stars?: number;
|
||||
discoveredVia: string;
|
||||
}> = new Map();
|
||||
|
||||
// 1. Awesome lists (fast, high yield)
|
||||
try {
|
||||
const awesomeResult = await this.runAwesomeListStrategy();
|
||||
stats.byStrategy['awesome-list'] = {
|
||||
repos: awesomeResult.repos.length,
|
||||
skills: 0,
|
||||
};
|
||||
for (const repo of awesomeResult.repos) {
|
||||
const key = `${repo.owner}/${repo.repo}`.toLowerCase();
|
||||
if (!allRepos.has(key)) {
|
||||
allRepos.set(key, repo);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Awesome list strategy failed:', error);
|
||||
}
|
||||
|
||||
// 2. Topic search (medium speed, medium yield)
|
||||
try {
|
||||
const topicResult = await this.runTopicSearchStrategy();
|
||||
stats.byStrategy['topic-search'] = {
|
||||
repos: topicResult.repos.length,
|
||||
skills: 0,
|
||||
};
|
||||
for (const repo of topicResult.repos) {
|
||||
const key = `${repo.owner}/${repo.repo}`.toLowerCase();
|
||||
if (!allRepos.has(key)) {
|
||||
allRepos.set(key, repo);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Topic search strategy failed:', error);
|
||||
}
|
||||
|
||||
// 3. Fork network (for high-star repos from above)
|
||||
try {
|
||||
const highStarRepos = Array.from(allRepos.values())
|
||||
.filter((r) => (r.stars || 0) >= 10)
|
||||
.slice(0, 50); // Top 50 repos by stars
|
||||
|
||||
if (highStarRepos.length > 0) {
|
||||
const forkResult = await this.runForkNetworkStrategy(
|
||||
highStarRepos.map((r) => ({ owner: r.owner, repo: r.repo }))
|
||||
);
|
||||
stats.byStrategy['fork-network'] = {
|
||||
repos: forkResult.repos.length,
|
||||
skills: 0,
|
||||
};
|
||||
for (const repo of forkResult.repos) {
|
||||
const key = `${repo.owner}/${repo.repo}`.toLowerCase();
|
||||
if (!allRepos.has(key)) {
|
||||
allRepos.set(key, repo);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Fork network strategy failed:', error);
|
||||
}
|
||||
|
||||
stats.totalReposDiscovered = allRepos.size;
|
||||
stats.duration = Date.now() - startTime;
|
||||
|
||||
console.log('\n=== Discovery Summary ===');
|
||||
console.log(`Total unique repositories: ${allRepos.size}`);
|
||||
console.log(`Duration: ${(stats.duration / 1000).toFixed(1)}s`);
|
||||
console.log('By strategy:');
|
||||
for (const [strategy, data] of Object.entries(stats.byStrategy)) {
|
||||
console.log(` ${strategy}: ${data.repos} repos`);
|
||||
}
|
||||
|
||||
return {
|
||||
repos: Array.from(allRepos.values()),
|
||||
stats,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export function createStrategyOrchestrator(tokenManager?: TokenManager | string): StrategyOrchestrator {
|
||||
if (typeof tokenManager === 'string') {
|
||||
return new StrategyOrchestrator(new TokenManager([tokenManager]));
|
||||
}
|
||||
return new StrategyOrchestrator(tokenManager);
|
||||
}
|
||||
|
||||
// Re-export strategy creators
|
||||
export { createAwesomeListCrawler } from './awesome-list.js';
|
||||
export { createTopicSearchCrawler } from './topic-search.js';
|
||||
export { createDeepScanCrawler } from './deep-scan.js';
|
||||
export { createForkNetworkCrawler } from './fork-network.js';
|
||||
|
||||
// Re-export types
|
||||
export type { RepoReference } from './awesome-list.js';
|
||||
export type { RepoResult } from './topic-search.js';
|
||||
export type { ForkInfo } from './fork-network.js';
|
||||
314
services/indexer/src/strategies/topic-search.ts
Normal file
314
services/indexer/src/strategies/topic-search.ts
Normal file
@@ -0,0 +1,314 @@
|
||||
import type { Octokit } from '@octokit/rest';
|
||||
import { TokenManager } from '../token-manager.js';import { OctokitPool } from '../octokit-pool.js';
|
||||
|
||||
export interface RepoResult {
|
||||
owner: string;
|
||||
repo: string;
|
||||
stars: number;
|
||||
forks: number;
|
||||
description: string | null;
|
||||
topics: string[];
|
||||
defaultBranch: string;
|
||||
isArchived: boolean;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Topics that indicate a repository might contain skills
|
||||
* Includes both compound topics (claude-skills) and simple topics (skill)
|
||||
*/
|
||||
const SKILL_TOPICS = [
|
||||
// Compound skill topics
|
||||
'claude-skills',
|
||||
'agent-skills',
|
||||
'ai-skills',
|
||||
'claude-code',
|
||||
'codex-skills',
|
||||
'copilot-skills',
|
||||
'skill-md',
|
||||
'anthropic-skills',
|
||||
'claude-code-skills',
|
||||
'ai-agent-skills',
|
||||
// Simple skill topics (for repos like openclaw/skills with topic "skill")
|
||||
'skill',
|
||||
'skills',
|
||||
// Platform-specific
|
||||
'clawdhub',
|
||||
'clawdbot',
|
||||
'skillhub',
|
||||
// AI agent related
|
||||
'ai-agent',
|
||||
'claude-agent',
|
||||
'anthropic',
|
||||
'llm-skills',
|
||||
'mcp-skills',
|
||||
// Cursor-specific
|
||||
'cursor-rules',
|
||||
'cursorrules',
|
||||
// Windsurf-specific
|
||||
'windsurf-rules',
|
||||
'windsurfrules',
|
||||
// Copilot-specific
|
||||
'copilot-instructions',
|
||||
'github-copilot',
|
||||
// Codex-specific
|
||||
'codex-agent',
|
||||
'openai-codex',
|
||||
];
|
||||
|
||||
/**
|
||||
* Search queries for repository descriptions and READMEs
|
||||
*/
|
||||
const REPO_SEARCH_QUERIES = [
|
||||
'SKILL.md in:readme',
|
||||
'claude skills in:description',
|
||||
'agent skills in:description',
|
||||
'"agent skill" in:readme',
|
||||
'claude code skill in:readme',
|
||||
'codex skill in:readme',
|
||||
'skill.md file in:readme',
|
||||
// Additional queries for platforms like clawdhub
|
||||
'clawdhub in:description',
|
||||
'clawdbot in:description',
|
||||
'claude code skills in:description',
|
||||
'agent skill in:name',
|
||||
'skills archive in:description',
|
||||
'anthropic skills in:description',
|
||||
'.cursorrules in:readme',
|
||||
'cursor rules in:description',
|
||||
'AGENTS.md in:readme',
|
||||
'codex agents in:description',
|
||||
'copilot-instructions in:readme',
|
||||
'windsurf rules in:description',
|
||||
'.windsurfrules in:readme',
|
||||
];
|
||||
|
||||
/**
|
||||
* Topic Search Strategy
|
||||
* Discovers repositories using GitHub's topic and description search
|
||||
*/
|
||||
export class TopicSearchCrawler {
|
||||
private octokitPool: OctokitPool;
|
||||
private tokenManager: TokenManager;
|
||||
|
||||
constructor(tokenManager?: TokenManager) {
|
||||
this.tokenManager = tokenManager || TokenManager.getInstance();
|
||||
this.octokitPool = new OctokitPool(this.tokenManager);
|
||||
}
|
||||
|
||||
private async getOctokit(): Promise<Octokit> {
|
||||
return this.octokitPool.getBestInstance();
|
||||
}
|
||||
|
||||
private getCurrentToken(): string {
|
||||
return this.tokenManager.getBestToken();
|
||||
}
|
||||
|
||||
/**
|
||||
* Search by all known skill-related topics
|
||||
*/
|
||||
async searchByTopics(): Promise<RepoResult[]> {
|
||||
const allRepos: RepoResult[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
console.log(`Searching by ${SKILL_TOPICS.length} skill-related topics...`);
|
||||
|
||||
for (const topic of SKILL_TOPICS) {
|
||||
try {
|
||||
console.log(` Searching topic: ${topic}`);
|
||||
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const response = await octokit.search.repos({
|
||||
q: `topic:${topic}`,
|
||||
sort: 'stars',
|
||||
order: 'desc',
|
||||
per_page: 100,
|
||||
});
|
||||
|
||||
this.octokitPool.updateStats(token, response.headers);
|
||||
console.log(` Found ${response.data.total_count} repos`);
|
||||
|
||||
for (const repo of response.data.items) {
|
||||
if (!repo.owner) continue;
|
||||
const key = `${repo.owner.login}/${repo.name}`.toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
allRepos.push({
|
||||
owner: repo.owner.login,
|
||||
repo: repo.name,
|
||||
stars: repo.stargazers_count,
|
||||
forks: repo.forks_count,
|
||||
description: repo.description,
|
||||
topics: repo.topics || [],
|
||||
defaultBranch: repo.default_branch,
|
||||
isArchived: repo.archived,
|
||||
updatedAt: repo.updated_at,
|
||||
});
|
||||
}
|
||||
|
||||
// Paginate if more results
|
||||
if (response.data.total_count > 100) {
|
||||
const additionalRepos = await this.paginateSearch(`topic:${topic}`, seen, 5);
|
||||
allRepos.push(...additionalRepos);
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(` Topic search failed for ${topic}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Topic search found ${allRepos.length} unique repositories`);
|
||||
return allRepos;
|
||||
}
|
||||
|
||||
/**
|
||||
* Search by description and README content
|
||||
*/
|
||||
async searchByDescription(): Promise<RepoResult[]> {
|
||||
const allRepos: RepoResult[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
console.log(`Searching by ${REPO_SEARCH_QUERIES.length} description queries...`);
|
||||
|
||||
for (const query of REPO_SEARCH_QUERIES) {
|
||||
try {
|
||||
console.log(` Query: ${query}`);
|
||||
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const response = await octokit.search.repos({
|
||||
q: query,
|
||||
sort: 'stars',
|
||||
order: 'desc',
|
||||
per_page: 100,
|
||||
});
|
||||
|
||||
this.octokitPool.updateStats(token, response.headers);
|
||||
console.log(` Found ${response.data.total_count} repos`);
|
||||
|
||||
for (const repo of response.data.items) {
|
||||
if (!repo.owner) continue;
|
||||
const key = `${repo.owner.login}/${repo.name}`.toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
allRepos.push({
|
||||
owner: repo.owner.login,
|
||||
repo: repo.name,
|
||||
stars: repo.stargazers_count,
|
||||
forks: repo.forks_count,
|
||||
description: repo.description,
|
||||
topics: repo.topics || [],
|
||||
defaultBranch: repo.default_branch,
|
||||
isArchived: repo.archived,
|
||||
updatedAt: repo.updated_at,
|
||||
});
|
||||
}
|
||||
|
||||
// Paginate if more results
|
||||
if (response.data.total_count > 100) {
|
||||
const additionalRepos = await this.paginateSearch(query, seen, 5);
|
||||
allRepos.push(...additionalRepos);
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(` Description search failed for "${query}":`, error);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Description search found ${allRepos.length} unique repositories`);
|
||||
return allRepos;
|
||||
}
|
||||
|
||||
/**
|
||||
* Paginate through search results
|
||||
*/
|
||||
private async paginateSearch(
|
||||
query: string,
|
||||
seen: Set<string>,
|
||||
maxPages: number
|
||||
): Promise<RepoResult[]> {
|
||||
const repos: RepoResult[] = [];
|
||||
|
||||
for (let page = 2; page <= maxPages; page++) {
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
|
||||
const response = await octokit.search.repos({
|
||||
q: query,
|
||||
sort: 'stars',
|
||||
order: 'desc',
|
||||
per_page: 100,
|
||||
page,
|
||||
});
|
||||
|
||||
this.octokitPool.updateStats(token, response.headers);
|
||||
|
||||
if (response.data.items.length === 0) break;
|
||||
|
||||
for (const repo of response.data.items) {
|
||||
if (!repo.owner) continue;
|
||||
const key = `${repo.owner.login}/${repo.name}`.toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
repos.push({
|
||||
owner: repo.owner.login,
|
||||
repo: repo.name,
|
||||
stars: repo.stargazers_count,
|
||||
forks: repo.forks_count,
|
||||
description: repo.description,
|
||||
topics: repo.topics || [],
|
||||
defaultBranch: repo.default_branch,
|
||||
isArchived: repo.archived,
|
||||
updatedAt: repo.updated_at,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
// Stop on 422 (beyond 1000 results)
|
||||
if (this.isBeyondResultsLimit(error)) break;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
return repos;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run all topic and description searches
|
||||
*/
|
||||
async discoverAll(): Promise<RepoResult[]> {
|
||||
const topicRepos = await this.searchByTopics();
|
||||
const descRepos = await this.searchByDescription();
|
||||
|
||||
// Merge and deduplicate
|
||||
const seen = new Set<string>();
|
||||
const allRepos: RepoResult[] = [];
|
||||
|
||||
for (const repo of [...topicRepos, ...descRepos]) {
|
||||
const key = `${repo.owner}/${repo.repo}`.toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
allRepos.push(repo);
|
||||
}
|
||||
|
||||
console.log(`Total unique repositories discovered: ${allRepos.length}`);
|
||||
return allRepos;
|
||||
}
|
||||
|
||||
private isBeyondResultsLimit(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Error &&
|
||||
'status' in error &&
|
||||
(error as { status: number }).status === 422
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function createTopicSearchCrawler(tokenManager?: TokenManager | string): TopicSearchCrawler {
|
||||
if (typeof tokenManager === 'string') {
|
||||
return new TopicSearchCrawler(new TokenManager([tokenManager]));
|
||||
}
|
||||
return new TopicSearchCrawler(tokenManager);
|
||||
}
|
||||
239
services/indexer/src/token-manager.ts
Normal file
239
services/indexer/src/token-manager.ts
Normal file
@@ -0,0 +1,239 @@
|
||||
/**
|
||||
* GitHub Token Manager
|
||||
* Manages multiple GitHub tokens for rate limit rotation
|
||||
*/
|
||||
|
||||
import { Octokit } from '@octokit/rest';
|
||||
|
||||
export interface TokenInfo {
|
||||
token: string;
|
||||
name: string;
|
||||
remaining: number;
|
||||
reset: number; // Unix timestamp in milliseconds
|
||||
limit: number;
|
||||
lastUsed: number;
|
||||
isExhausted: boolean;
|
||||
}
|
||||
|
||||
export interface RateLimitStatus {
|
||||
totalTokens: number;
|
||||
availableTokens: number;
|
||||
globalRemaining: number;
|
||||
nextReset: number;
|
||||
tokens: TokenInfo[];
|
||||
}
|
||||
|
||||
export class TokenManager {
|
||||
private static instance: TokenManager | null = null;
|
||||
private tokens: TokenInfo[] = [];
|
||||
private octokit: Map<string, Octokit> = new Map();
|
||||
|
||||
constructor(tokens?: string[], names?: string[]) {
|
||||
if (!tokens || tokens.length === 0) {
|
||||
tokens = this.parseTokensFromEnv();
|
||||
}
|
||||
|
||||
const tokenNames = names || this.generateTokenNames(tokens.length);
|
||||
|
||||
this.tokens = tokens.map((token, i) => ({
|
||||
token,
|
||||
name: tokenNames[i],
|
||||
remaining: 5000, // Default GitHub authenticated limit
|
||||
reset: Date.now() + 3600000, // 1 hour from now
|
||||
limit: 5000,
|
||||
lastUsed: 0,
|
||||
isExhausted: false,
|
||||
}));
|
||||
|
||||
console.log(`TokenManager initialized with ${this.tokens.length} token(s)`);
|
||||
|
||||
// Refresh all tokens from GitHub API to get accurate rate limits
|
||||
this.refreshAllTokens().catch((error) => {
|
||||
console.warn('Failed to refresh tokens on init:', error);
|
||||
});
|
||||
}
|
||||
|
||||
private async refreshAllTokens(): Promise<void> {
|
||||
for (const tokenInfo of this.tokens) {
|
||||
await this.refreshRateLimit(tokenInfo.token);
|
||||
}
|
||||
}
|
||||
|
||||
static getInstance(): TokenManager {
|
||||
if (!this.instance) {
|
||||
this.instance = new TokenManager();
|
||||
}
|
||||
return this.instance;
|
||||
}
|
||||
|
||||
private parseTokensFromEnv(): string[] {
|
||||
// Try multi-token format first
|
||||
const tokensEnv = process.env.GITHUB_TOKENS;
|
||||
if (tokensEnv) {
|
||||
const tokens = tokensEnv.split(',').map((t) => t.trim()).filter(Boolean);
|
||||
if (tokens.length > 0) {
|
||||
return tokens;
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to single token (backward compatible)
|
||||
const singleToken = process.env.GITHUB_TOKEN;
|
||||
if (singleToken) {
|
||||
return [singleToken];
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
'No GitHub tokens configured. Set GITHUB_TOKEN or GITHUB_TOKENS environment variable'
|
||||
);
|
||||
}
|
||||
|
||||
private generateTokenNames(count: number): string[] {
|
||||
const namesEnv = process.env.GITHUB_TOKEN_NAMES;
|
||||
if (namesEnv) {
|
||||
const names = namesEnv.split(',').map((n) => n.trim());
|
||||
if (names.length === count) {
|
||||
return names;
|
||||
}
|
||||
}
|
||||
// Auto-generate names
|
||||
return Array.from({ length: count }, (_, i) => `token-${i + 1}`);
|
||||
}
|
||||
|
||||
getBestToken(): string {
|
||||
// Find token with highest remaining requests
|
||||
const available = this.tokens
|
||||
.filter((t) => !t.isExhausted)
|
||||
.sort((a, b) => b.remaining - a.remaining);
|
||||
|
||||
if (available.length > 0) {
|
||||
const best = available[0];
|
||||
best.lastUsed = Date.now();
|
||||
return best.token;
|
||||
}
|
||||
|
||||
// All exhausted - return token with earliest reset
|
||||
const earliest = this.tokens.sort((a, b) => a.reset - b.reset)[0];
|
||||
return earliest.token;
|
||||
}
|
||||
|
||||
updateTokenStats(token: string, headers: Record<string, unknown>): void {
|
||||
const tokenInfo = this.tokens.find((t) => t.token === token);
|
||||
if (!tokenInfo) return;
|
||||
|
||||
const remaining = headers['x-ratelimit-remaining'];
|
||||
const reset = headers['x-ratelimit-reset'];
|
||||
const limit = headers['x-ratelimit-limit'];
|
||||
|
||||
// GitHub Code Search API returns its own rate limit headers (limit=10 or 30)
|
||||
// which differ from the REST API limit (5000). Only update token stats
|
||||
// from REST API responses to avoid corrupting the primary rate limit tracking.
|
||||
const parsedLimit = typeof limit === 'string' ? parseInt(limit, 10) : 0;
|
||||
if (parsedLimit > 0 && parsedLimit < 100) {
|
||||
// Secondary rate limit (code search, etc.) - skip updating primary stats
|
||||
return;
|
||||
}
|
||||
|
||||
if (typeof remaining === 'string') {
|
||||
tokenInfo.remaining = parseInt(remaining, 10);
|
||||
tokenInfo.isExhausted = tokenInfo.remaining < 10;
|
||||
}
|
||||
if (typeof reset === 'string') {
|
||||
tokenInfo.reset = parseInt(reset, 10) * 1000;
|
||||
}
|
||||
if (parsedLimit > 0) {
|
||||
tokenInfo.limit = parsedLimit;
|
||||
}
|
||||
|
||||
if (tokenInfo.remaining % 100 === 0 || tokenInfo.isExhausted) {
|
||||
console.log(`[${tokenInfo.name}] ${tokenInfo.remaining}/${tokenInfo.limit} requests remaining`);
|
||||
}
|
||||
}
|
||||
|
||||
async checkAndRotate(): Promise<string> {
|
||||
const current = this.getBestToken();
|
||||
const currentInfo = this.tokens.find((t) => t.token === current);
|
||||
|
||||
if (!currentInfo) return current;
|
||||
|
||||
// If current token is exhausted, try to rotate
|
||||
if (currentInfo.isExhausted) {
|
||||
console.log(`[${currentInfo.name}] Exhausted, checking other tokens...`);
|
||||
|
||||
// Check if any other tokens are available
|
||||
const available = this.tokens.find((t) => !t.isExhausted && t.token !== current);
|
||||
|
||||
if (available) {
|
||||
this.logRotation(currentInfo.name, available.name, 'Token exhausted');
|
||||
return available.token;
|
||||
}
|
||||
|
||||
// All tokens exhausted - wait for earliest reset
|
||||
const waitTime = Math.max(0, currentInfo.reset - Date.now()) + 1000;
|
||||
console.warn(
|
||||
`All tokens exhausted. Waiting ${Math.ceil(waitTime / 1000)}s until [${currentInfo.name}] resets...`
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, waitTime));
|
||||
|
||||
// Refresh rate limit after waiting
|
||||
await this.refreshRateLimit(current);
|
||||
}
|
||||
|
||||
return current;
|
||||
}
|
||||
|
||||
async refreshRateLimit(token: string): Promise<void> {
|
||||
const tokenInfo = this.tokens.find((t) => t.token === token);
|
||||
if (!tokenInfo) return;
|
||||
|
||||
try {
|
||||
let octokit = this.octokit.get(token);
|
||||
if (!octokit) {
|
||||
octokit = new Octokit({
|
||||
auth: token,
|
||||
userAgent: 'SkillHub-Indexer/1.0',
|
||||
});
|
||||
this.octokit.set(token, octokit);
|
||||
}
|
||||
|
||||
const response = await octokit.rateLimit.get();
|
||||
const core = response.data.resources.core;
|
||||
|
||||
tokenInfo.remaining = core.remaining;
|
||||
tokenInfo.reset = core.reset * 1000;
|
||||
tokenInfo.limit = core.limit;
|
||||
tokenInfo.isExhausted = core.remaining < 10;
|
||||
|
||||
console.log(
|
||||
`[${tokenInfo.name}] Refreshed: ${core.remaining}/${core.limit} (resets at ${new Date(tokenInfo.reset).toLocaleTimeString()})`
|
||||
);
|
||||
} catch (error) {
|
||||
console.error(`Failed to refresh rate limit for [${tokenInfo.name}]:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
getStatus(): RateLimitStatus {
|
||||
const availableTokens = this.tokens.filter((t) => !t.isExhausted);
|
||||
const globalRemaining = this.tokens.reduce((sum, t) => sum + t.remaining, 0);
|
||||
const nextReset = Math.min(...this.tokens.map((t) => t.reset));
|
||||
|
||||
return {
|
||||
totalTokens: this.tokens.length,
|
||||
availableTokens: availableTokens.length,
|
||||
globalRemaining,
|
||||
nextReset,
|
||||
tokens: [...this.tokens],
|
||||
};
|
||||
}
|
||||
|
||||
private logRotation(from: string, to: string, reason: string): void {
|
||||
console.log(`
|
||||
═══════════════════════════════════════
|
||||
TOKEN ROTATION
|
||||
From: [${from}]
|
||||
To: [${to}]
|
||||
Reason: ${reason}
|
||||
Time: ${new Date().toISOString()}
|
||||
═══════════════════════════════════════
|
||||
`);
|
||||
}
|
||||
}
|
||||
649
services/indexer/src/worker.ts
Normal file
649
services/indexer/src/worker.ts
Normal file
@@ -0,0 +1,649 @@
|
||||
import type { Job } from 'bullmq';
|
||||
import { Worker } from 'bullmq';
|
||||
import pLimit from 'p-limit';
|
||||
import { createDb, type Database, discoveredRepoQueries, awesomeListQueries, addRequestQueries, skillQueries } from '@skillhub/db';
|
||||
import { GitHubCrawler, createCrawler } from './crawler.js';
|
||||
import type { IndexJobData, IndexJobResult } from './queue.js';
|
||||
import { setupRecurringJobs } from './queue.js';
|
||||
import { logMeilisearchStatus } from './meilisearch-sync.js';
|
||||
import { indexSkill } from './skill-indexer.js';
|
||||
import { createStrategyOrchestrator, createDeepScanCrawler, createAwesomeListCrawler } from './strategies/index.js';
|
||||
|
||||
const QUEUE_NAME = 'skill-indexing';
|
||||
const CONCURRENCY = 5;
|
||||
|
||||
let db: Database | null = null;
|
||||
|
||||
function getRedisConnection(): { host: string; port: number; password?: string; username?: string } {
|
||||
const redisUrl = process.env.REDIS_URL;
|
||||
if (redisUrl) {
|
||||
const url = new URL(redisUrl);
|
||||
const connection: {
|
||||
host: string;
|
||||
port: number;
|
||||
password?: string;
|
||||
username?: string;
|
||||
} = {
|
||||
host: url.hostname,
|
||||
port: parseInt(url.port) || 6379,
|
||||
};
|
||||
// Add authentication if present in URL
|
||||
if (url.password) {
|
||||
connection.password = decodeURIComponent(url.password);
|
||||
}
|
||||
if (url.username && url.username !== 'default') {
|
||||
connection.username = decodeURIComponent(url.username);
|
||||
}
|
||||
return connection;
|
||||
}
|
||||
return {
|
||||
host: process.env.REDIS_HOST || 'localhost',
|
||||
port: parseInt(process.env.REDIS_PORT || '6379'),
|
||||
};
|
||||
}
|
||||
|
||||
function getDb(): Database {
|
||||
if (!db) {
|
||||
db = createDb(process.env.DATABASE_URL);
|
||||
}
|
||||
return db;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the indexing worker
|
||||
*/
|
||||
export function startWorker(): Worker<IndexJobData, IndexJobResult> {
|
||||
const worker = new Worker<IndexJobData, IndexJobResult>(
|
||||
QUEUE_NAME,
|
||||
async (job: Job<IndexJobData, IndexJobResult>) => {
|
||||
const startTime = Date.now();
|
||||
|
||||
console.log(`Processing job ${job.id}: ${job.data.type}`);
|
||||
|
||||
try {
|
||||
switch (job.data.type) {
|
||||
case 'full-crawl':
|
||||
return await processFullCrawl(job);
|
||||
|
||||
case 'incremental':
|
||||
return await processIncrementalCrawl(job);
|
||||
|
||||
case 'index-skill':
|
||||
return await processSkillIndex(job);
|
||||
|
||||
case 'discover-repos':
|
||||
return await processDiscoverRepos(job);
|
||||
|
||||
case 'awesome-lists':
|
||||
return await processAwesomeLists(job);
|
||||
|
||||
case 'deep-scan':
|
||||
return await processDeepScan(job);
|
||||
|
||||
case 'full-enhanced':
|
||||
return await processFullEnhanced(job);
|
||||
|
||||
case 'process-add-requests':
|
||||
return await processAddRequests(job);
|
||||
|
||||
|
||||
default:
|
||||
throw new Error(`Unknown job type: ${job.data.type}`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Job ${job.id} failed:`, error);
|
||||
throw error;
|
||||
} finally {
|
||||
const duration = Date.now() - startTime;
|
||||
console.log(`Job ${job.id} completed in ${duration}ms`);
|
||||
}
|
||||
},
|
||||
{
|
||||
connection: getRedisConnection(),
|
||||
concurrency: CONCURRENCY,
|
||||
}
|
||||
);
|
||||
|
||||
worker.on('completed', (job, result) => {
|
||||
console.log(`Job ${job.id} completed:`, result);
|
||||
});
|
||||
|
||||
worker.on('failed', (job, error) => {
|
||||
console.error(`Job ${job?.id} failed:`, error.message);
|
||||
});
|
||||
|
||||
worker.on('error', (error) => {
|
||||
console.error('Worker error:', error);
|
||||
});
|
||||
|
||||
console.log(`Worker started with concurrency ${CONCURRENCY}`);
|
||||
|
||||
return worker;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a full crawl job
|
||||
*/
|
||||
async function processFullCrawl(
|
||||
job: Job<IndexJobData, IndexJobResult>
|
||||
): Promise<IndexJobResult> {
|
||||
const crawler = new GitHubCrawler();
|
||||
const options = job.data.options || {};
|
||||
|
||||
// Discover all skill repositories
|
||||
await job.updateProgress(10);
|
||||
console.log('Discovering skill repositories...');
|
||||
|
||||
const sources = await crawler.discoverSkillRepos({
|
||||
minStars: options.minStars ?? 2,
|
||||
maxPages: 50,
|
||||
});
|
||||
|
||||
console.log(`Discovered ${sources.length} potential skills`);
|
||||
await job.updateProgress(30);
|
||||
|
||||
// Index each skill with rate limiting
|
||||
const limit = pLimit(3); // Process 3 at a time
|
||||
const results = { indexed: 0, failed: 0, skipped: 0 };
|
||||
|
||||
const indexPromises = sources.map((source, index) =>
|
||||
limit(async () => {
|
||||
try {
|
||||
const skillId = await indexSkill(crawler, source, options.force);
|
||||
if (skillId) {
|
||||
results.indexed++;
|
||||
} else {
|
||||
results.skipped++;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to index ${source.owner}/${source.repo}:`, error);
|
||||
results.failed++;
|
||||
}
|
||||
|
||||
// Update progress
|
||||
const progress = 30 + Math.floor((index / sources.length) * 60);
|
||||
await job.updateProgress(progress);
|
||||
})
|
||||
);
|
||||
|
||||
await Promise.all(indexPromises);
|
||||
await job.updateProgress(100);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
stats: {
|
||||
discovered: sources.length,
|
||||
indexed: results.indexed,
|
||||
failed: results.failed,
|
||||
duration: Date.now() - job.processedOn!,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Process an incremental crawl job
|
||||
*/
|
||||
async function processIncrementalCrawl(
|
||||
job: Job<IndexJobData, IndexJobResult>
|
||||
): Promise<IndexJobResult> {
|
||||
const crawler = new GitHubCrawler();
|
||||
const options = job.data.options || {};
|
||||
|
||||
const updatedAfter = options.updatedAfter
|
||||
? new Date(options.updatedAfter)
|
||||
: new Date(Date.now() - 24 * 60 * 60 * 1000);
|
||||
|
||||
await job.updateProgress(10);
|
||||
console.log(`Looking for skills updated since ${updatedAfter.toISOString()}`);
|
||||
|
||||
const sources = await crawler.discoverSkillRepos({
|
||||
minStars: 1,
|
||||
updatedAfter,
|
||||
maxPages: 20,
|
||||
});
|
||||
|
||||
console.log(`Found ${sources.length} recently updated skills`);
|
||||
await job.updateProgress(30);
|
||||
|
||||
const limit = pLimit(5);
|
||||
const results = { indexed: 0, failed: 0 };
|
||||
|
||||
const indexPromises = sources.map((source, index) =>
|
||||
limit(async () => {
|
||||
try {
|
||||
await indexSkill(crawler, source, true);
|
||||
results.indexed++;
|
||||
} catch (error) {
|
||||
console.error(`Failed to index ${source.owner}/${source.repo}:`, error);
|
||||
results.failed++;
|
||||
}
|
||||
|
||||
const progress = 30 + Math.floor((index / sources.length) * 60);
|
||||
await job.updateProgress(progress);
|
||||
})
|
||||
);
|
||||
|
||||
await Promise.all(indexPromises);
|
||||
await job.updateProgress(100);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
stats: {
|
||||
discovered: sources.length,
|
||||
indexed: results.indexed,
|
||||
failed: results.failed,
|
||||
duration: Date.now() - job.processedOn!,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single skill index job
|
||||
*/
|
||||
async function processSkillIndex(
|
||||
job: Job<IndexJobData, IndexJobResult>
|
||||
): Promise<IndexJobResult> {
|
||||
const source = job.data.source;
|
||||
if (!source) {
|
||||
throw new Error('Missing skill source');
|
||||
}
|
||||
|
||||
const crawler = new GitHubCrawler();
|
||||
const force = job.data.options?.force ?? false;
|
||||
|
||||
await job.updateProgress(10);
|
||||
|
||||
try {
|
||||
const skillId = await indexSkill(crawler, source, force);
|
||||
|
||||
if (skillId) {
|
||||
return {
|
||||
success: true,
|
||||
skillId,
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
success: true,
|
||||
skillId: `${source.owner}/${source.repo}/${source.path}`,
|
||||
stats: { indexed: 0 },
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process discover-repos job - run all discovery strategies
|
||||
*/
|
||||
async function processDiscoverRepos(
|
||||
job: Job<IndexJobData, IndexJobResult>
|
||||
): Promise<IndexJobResult> {
|
||||
const database = getDb();
|
||||
const orchestrator = createStrategyOrchestrator();
|
||||
|
||||
await job.updateProgress(10);
|
||||
console.log('Running all discovery strategies...');
|
||||
|
||||
const { repos: discoveredRepos, stats: discoverStats } = await orchestrator.runAllStrategies();
|
||||
|
||||
await job.updateProgress(50);
|
||||
console.log(`Discovered ${discoveredRepos.length} repositories, saving to database...`);
|
||||
|
||||
let savedRepos = 0;
|
||||
for (const repo of discoveredRepos) {
|
||||
try {
|
||||
await discoveredRepoQueries.upsert(database, {
|
||||
id: `${repo.owner}/${repo.repo}`,
|
||||
owner: repo.owner,
|
||||
repo: repo.repo,
|
||||
discoveredVia: repo.discoveredVia,
|
||||
githubStars: repo.stars,
|
||||
});
|
||||
savedRepos++;
|
||||
} catch {
|
||||
// Skip duplicates
|
||||
}
|
||||
}
|
||||
|
||||
await job.updateProgress(100);
|
||||
console.log(`Saved ${savedRepos} new repositories`);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
stats: {
|
||||
discovered: discoveredRepos.length,
|
||||
indexed: savedRepos,
|
||||
duration: discoverStats.duration,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Process awesome-lists job - crawl curated lists for skill repos
|
||||
*/
|
||||
async function processAwesomeLists(
|
||||
job: Job<IndexJobData, IndexJobResult>
|
||||
): Promise<IndexJobResult> {
|
||||
const database = getDb();
|
||||
const awesomeCrawler = createAwesomeListCrawler();
|
||||
|
||||
await job.updateProgress(10);
|
||||
console.log('Running awesome list discovery...');
|
||||
|
||||
// Save known lists to DB
|
||||
for (const list of awesomeCrawler.getKnownLists()) {
|
||||
await awesomeListQueries.upsert(database, {
|
||||
id: `${list.owner}/${list.repo}`,
|
||||
owner: list.owner,
|
||||
repo: list.repo,
|
||||
name: `${list.owner}/${list.repo}`,
|
||||
});
|
||||
}
|
||||
|
||||
await job.updateProgress(30);
|
||||
const listResults = await awesomeCrawler.crawlAllLists();
|
||||
let awesomeTotal = 0;
|
||||
|
||||
await job.updateProgress(60);
|
||||
for (const [listId, repoRefs] of listResults.entries()) {
|
||||
console.log(`${listId}: ${repoRefs.length} repos`);
|
||||
|
||||
// Update list stats
|
||||
await awesomeListQueries.markParsed(database, listId, repoRefs.length);
|
||||
|
||||
// Save repos
|
||||
for (const ref of repoRefs) {
|
||||
try {
|
||||
await discoveredRepoQueries.upsert(database, {
|
||||
id: `${ref.owner}/${ref.repo}`,
|
||||
owner: ref.owner,
|
||||
repo: ref.repo,
|
||||
discoveredVia: 'awesome-list',
|
||||
sourceUrl: `https://github.com/${listId}`,
|
||||
});
|
||||
awesomeTotal++;
|
||||
} catch {
|
||||
// Skip duplicates
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await job.updateProgress(100);
|
||||
console.log(`Saved ${awesomeTotal} repositories from awesome lists`);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
stats: {
|
||||
discovered: awesomeTotal,
|
||||
indexed: awesomeTotal,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Process deep-scan job - scan discovered repos for SKILL.md files
|
||||
*/
|
||||
async function processDeepScan(
|
||||
job: Job<IndexJobData, IndexJobResult>
|
||||
): Promise<IndexJobResult> {
|
||||
const database = getDb();
|
||||
const deepCrawler = createDeepScanCrawler();
|
||||
const skillCrawler = createCrawler();
|
||||
const options = job.data.options || {};
|
||||
const scanLimit = options.scanLimit || 100;
|
||||
|
||||
await job.updateProgress(10);
|
||||
console.log('Deep scanning discovered repositories...');
|
||||
|
||||
// Get repos that need scanning (never scanned or stale)
|
||||
const oneWeekAgo = new Date();
|
||||
oneWeekAgo.setDate(oneWeekAgo.getDate() - 7);
|
||||
const reposToScan = await discoveredRepoQueries.getNeedingScanning(database, oneWeekAgo, scanLimit);
|
||||
|
||||
if (reposToScan.length === 0) {
|
||||
console.log('No repositories need scanning');
|
||||
return { success: true, stats: { discovered: 0, indexed: 0 } };
|
||||
}
|
||||
|
||||
console.log(`Found ${reposToScan.length} repositories to scan`);
|
||||
|
||||
let scannedCount = 0;
|
||||
let skillsDiscovered = 0;
|
||||
let skillsIndexed = 0;
|
||||
|
||||
for (const repo of reposToScan) {
|
||||
try {
|
||||
console.log(`Scanning ${repo.owner}/${repo.repo}...`);
|
||||
const skills = await deepCrawler.scanRepository(repo.owner, repo.repo);
|
||||
|
||||
// Mark as scanned
|
||||
await discoveredRepoQueries.markScanned(
|
||||
database,
|
||||
repo.id,
|
||||
skills.length,
|
||||
skills.length > 0
|
||||
);
|
||||
|
||||
scannedCount++;
|
||||
skillsDiscovered += skills.length;
|
||||
|
||||
// If skills found, index them
|
||||
if (skills.length > 0) {
|
||||
console.log(` Found ${skills.length} skills, processing...`);
|
||||
for (const skillSource of skills) {
|
||||
try {
|
||||
const skillId = await indexSkill(skillCrawler, skillSource, false);
|
||||
if (skillId) {
|
||||
skillsIndexed++;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(` Failed to index ${skillSource.path}:`, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update progress
|
||||
const progress = 10 + Math.floor((scannedCount / reposToScan.length) * 85);
|
||||
await job.updateProgress(progress);
|
||||
} catch (error) {
|
||||
console.log(` Error scanning ${repo.owner}/${repo.repo}:`, error);
|
||||
await discoveredRepoQueries.markScanned(database, repo.id, 0, false, String(error));
|
||||
}
|
||||
}
|
||||
|
||||
await job.updateProgress(100);
|
||||
console.log(`Deep scan complete: ${scannedCount} repos, ${skillsDiscovered} skills found, ${skillsIndexed} indexed`);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
stats: {
|
||||
discovered: skillsDiscovered,
|
||||
indexed: skillsIndexed,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Process full-enhanced job - discovery + deep-scan + full-crawl
|
||||
*/
|
||||
async function processFullEnhanced(
|
||||
job: Job<IndexJobData, IndexJobResult>
|
||||
): Promise<IndexJobResult> {
|
||||
const startTime = Date.now();
|
||||
|
||||
// Step 1: Run discovery
|
||||
await job.updateProgress(5);
|
||||
console.log('Step 1/3: Running discovery strategies...');
|
||||
await processDiscoverRepos(job);
|
||||
|
||||
// Step 2: Run deep scan
|
||||
await job.updateProgress(35);
|
||||
console.log('Step 2/3: Running deep scan...');
|
||||
await processDeepScan(job);
|
||||
|
||||
// Step 3: Run full crawl
|
||||
await job.updateProgress(65);
|
||||
console.log('Step 3/3: Running full crawl...');
|
||||
const crawlResult = await processFullCrawl(job);
|
||||
|
||||
await job.updateProgress(100);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
stats: {
|
||||
...crawlResult.stats,
|
||||
duration: Date.now() - startTime,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Process add-requests job - index user-submitted skill requests
|
||||
*/
|
||||
async function processAddRequests(
|
||||
job: Job<IndexJobData, IndexJobResult>
|
||||
): Promise<IndexJobResult> {
|
||||
const database = getDb();
|
||||
const addCrawler = createCrawler();
|
||||
|
||||
await job.updateProgress(10);
|
||||
console.log('Processing pending add requests...');
|
||||
|
||||
// Get all pending add requests
|
||||
const pendingRequests = await addRequestQueries.getAllPending(database);
|
||||
|
||||
if (pendingRequests.length === 0) {
|
||||
console.log('No pending add requests found');
|
||||
return { success: true, stats: { discovered: 0, indexed: 0 } };
|
||||
}
|
||||
|
||||
console.log(`Found ${pendingRequests.length} pending request(s)`);
|
||||
|
||||
let processedCount = 0;
|
||||
let skillsIndexed = 0;
|
||||
let failedCount = 0;
|
||||
|
||||
for (const request of pendingRequests) {
|
||||
console.log(`Processing request ${request.id.slice(0, 8)}...`);
|
||||
|
||||
// Skip if no skill paths found
|
||||
if (!request.hasSkillMd || !request.skillPath) {
|
||||
await addRequestQueries.updateStatus(database, request.id, {
|
||||
status: 'approved',
|
||||
errorMessage: 'No SKILL.md found - requires manual review',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
// Parse repository URL
|
||||
const url = new URL(request.repositoryUrl);
|
||||
const pathParts = url.pathname.split('/').filter(Boolean);
|
||||
const owner = pathParts[0];
|
||||
const repo = pathParts[1];
|
||||
|
||||
if (!owner || !repo) {
|
||||
throw new Error('Invalid repository URL');
|
||||
}
|
||||
|
||||
// Get default branch
|
||||
const repoMeta = await addCrawler.getRepoMetadata(owner, repo);
|
||||
const branch = repoMeta.defaultBranch;
|
||||
|
||||
// Parse skill paths (comma-separated)
|
||||
const skillPaths = request.skillPath.split(',').map((p: string) => p.trim());
|
||||
const indexedSkillIds: string[] = [];
|
||||
const existingSkillIds: string[] = [];
|
||||
|
||||
for (const skillPath of skillPaths) {
|
||||
try {
|
||||
// Check if skill already exists
|
||||
const skillName = skillPath.split('/').pop() || 'skill';
|
||||
const potentialSkillId = `${owner}/${repo}/${skillName}`;
|
||||
const existingSkill = await skillQueries.getById(database, potentialSkillId);
|
||||
|
||||
if (existingSkill && !existingSkill.isBlocked) {
|
||||
existingSkillIds.push(potentialSkillId);
|
||||
continue;
|
||||
}
|
||||
|
||||
const skillId = await indexSkill(
|
||||
addCrawler,
|
||||
{ owner, repo, path: skillPath || '.', branch },
|
||||
true
|
||||
);
|
||||
|
||||
if (skillId) {
|
||||
indexedSkillIds.push(skillId);
|
||||
skillsIndexed++;
|
||||
}
|
||||
} catch (skillError) {
|
||||
console.error(` Failed to index ${skillPath}:`, skillError);
|
||||
}
|
||||
}
|
||||
|
||||
// Update request status
|
||||
const allSkillIds = [...indexedSkillIds, ...existingSkillIds];
|
||||
if (allSkillIds.length > 0) {
|
||||
await addRequestQueries.updateStatus(database, request.id, {
|
||||
status: 'indexed',
|
||||
indexedSkillId: allSkillIds.join(','),
|
||||
});
|
||||
} else {
|
||||
await addRequestQueries.updateStatus(database, request.id, {
|
||||
status: 'approved',
|
||||
errorMessage: 'Could not index any skills',
|
||||
});
|
||||
}
|
||||
|
||||
processedCount++;
|
||||
} catch (error) {
|
||||
failedCount++;
|
||||
await addRequestQueries.updateStatus(database, request.id, {
|
||||
status: 'approved',
|
||||
errorMessage: error instanceof Error ? error.message : 'Unknown error',
|
||||
});
|
||||
}
|
||||
|
||||
// Update progress
|
||||
const progress = 10 + Math.floor((processedCount / pendingRequests.length) * 85);
|
||||
await job.updateProgress(progress);
|
||||
}
|
||||
|
||||
await job.updateProgress(100);
|
||||
console.log(`Processed ${processedCount} requests, indexed ${skillsIndexed} skills, ${failedCount} failed`);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
stats: {
|
||||
discovered: pendingRequests.length,
|
||||
indexed: skillsIndexed,
|
||||
failed: failedCount,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
// Run worker when this file is executed
|
||||
console.log('Starting indexer worker...');
|
||||
logMeilisearchStatus();
|
||||
const worker = startWorker();
|
||||
|
||||
// Setup recurring jobs for automatic crawling
|
||||
setupRecurringJobs()
|
||||
.then(() => console.log('Recurring jobs initialized'))
|
||||
.catch((err) => console.error('Failed to setup recurring jobs:', err));
|
||||
|
||||
// Handle shutdown
|
||||
const shutdown = async () => {
|
||||
console.log('Shutting down worker...');
|
||||
await worker.close();
|
||||
process.exit(0);
|
||||
};
|
||||
|
||||
process.on('SIGTERM', shutdown);
|
||||
process.on('SIGINT', shutdown);
|
||||
9
services/indexer/tsconfig.json
Normal file
9
services/indexer/tsconfig.json
Normal file
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src"
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
39
services/indexer/tsup.config.ts
Normal file
39
services/indexer/tsup.config.ts
Normal file
@@ -0,0 +1,39 @@
|
||||
import { defineConfig } from 'tsup';
|
||||
|
||||
export default defineConfig({
|
||||
entry: ['src/index.ts', 'src/worker.ts', 'src/crawl.ts'],
|
||||
format: ['cjs'],
|
||||
target: 'node20',
|
||||
platform: 'node',
|
||||
splitting: false,
|
||||
sourcemap: true,
|
||||
clean: true,
|
||||
// Bundle all dependencies into the output
|
||||
noExternal: [/.*/],
|
||||
// Exclude Node.js built-in modules
|
||||
external: [
|
||||
'node:*',
|
||||
'fs',
|
||||
'path',
|
||||
'crypto',
|
||||
'os',
|
||||
'url',
|
||||
'util',
|
||||
'stream',
|
||||
'events',
|
||||
'buffer',
|
||||
'http',
|
||||
'https',
|
||||
'net',
|
||||
'tls',
|
||||
'dns',
|
||||
'child_process',
|
||||
'cluster',
|
||||
'worker_threads',
|
||||
'assert',
|
||||
'async_hooks',
|
||||
'string_decoder',
|
||||
'querystring',
|
||||
'zlib',
|
||||
],
|
||||
});
|
||||
Reference in New Issue
Block a user