Initial release v1.0.0
Open-source marketplace for AI Agent skills. Features: - Next.js 15 web app with i18n (en/fa) - CLI tool for skill installation (npx skillhub) - GitHub crawler/indexer with multi-strategy discovery - Security scanning for all indexed skills - Self-hostable with Docker Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
airano
203
services/indexer/src/strategies/awesome-list.ts
Normal file
203
services/indexer/src/strategies/awesome-list.ts
Normal file
@@ -0,0 +1,203 @@
|
||||
import type { Octokit } from '@octokit/rest';
|
||||
import { TokenManager } from '../token-manager.js';
|
||||
import { OctokitPool } from '../octokit-pool.js';
|
||||
|
||||
export interface RepoReference {
|
||||
owner: string;
|
||||
repo: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Known awesome lists that contain skill repositories
|
||||
*/
|
||||
const KNOWN_AWESOME_LISTS = [
|
||||
{ owner: 'travisvn', repo: 'awesome-claude-skills', readme: 'README.md' },
|
||||
{ owner: 'VoltAgent', repo: 'awesome-claude-skills', readme: 'README.md' },
|
||||
{ owner: 'ComposioHQ', repo: 'awesome-claude-skills', readme: 'README.md' },
|
||||
{ owner: 'skillmatic-ai', repo: 'awesome-agent-skills', readme: 'README.md' },
|
||||
{ owner: 'github', repo: 'awesome-copilot', readme: 'README.md' },
|
||||
];
|
||||
|
||||
/**
|
||||
* Awesome List Crawler Strategy
|
||||
* Parses README files from curated "awesome" lists to discover skill repositories
|
||||
*/
|
||||
export class AwesomeListCrawler {
|
||||
private octokitPool: OctokitPool;
|
||||
private tokenManager: TokenManager;
|
||||
|
||||
constructor(tokenManager?: TokenManager) {
|
||||
this.tokenManager = tokenManager || TokenManager.getInstance();
|
||||
this.octokitPool = new OctokitPool(this.tokenManager);
|
||||
}
|
||||
|
||||
private async getOctokit(): Promise<Octokit> {
|
||||
return this.octokitPool.getBestInstance();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all known awesome lists
|
||||
*/
|
||||
getKnownLists() {
|
||||
return KNOWN_AWESOME_LISTS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Crawl all known awesome lists
|
||||
*/
|
||||
async crawlAllLists(): Promise<Map<string, RepoReference[]>> {
|
||||
const results = new Map<string, RepoReference[]>();
|
||||
|
||||
for (const list of KNOWN_AWESOME_LISTS) {
|
||||
try {
|
||||
console.log(`Parsing awesome list: ${list.owner}/${list.repo}`);
|
||||
const repos = await this.parseAwesomeList(list.owner, list.repo, list.readme);
|
||||
results.set(`${list.owner}/${list.repo}`, repos);
|
||||
console.log(` Found ${repos.length} repository references`);
|
||||
} catch (error) {
|
||||
console.warn(`Failed to parse ${list.owner}/${list.repo}:`, error);
|
||||
results.set(`${list.owner}/${list.repo}`, []);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an awesome list README to extract GitHub repository references
|
||||
*/
|
||||
async parseAwesomeList(
|
||||
owner: string,
|
||||
repo: string,
|
||||
readmePath = 'README.md'
|
||||
): Promise<RepoReference[]> {
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const response = await octokit.repos.getContent({
|
||||
owner,
|
||||
repo,
|
||||
path: readmePath,
|
||||
});
|
||||
|
||||
if (!('content' in response.data)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const content = Buffer.from(response.data.content, 'base64').toString('utf-8');
|
||||
return this.extractGitHubRepos(content);
|
||||
} catch (error) {
|
||||
console.warn(`Failed to fetch ${owner}/${repo}/${readmePath}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract GitHub repository references from markdown content
|
||||
*/
|
||||
extractGitHubRepos(content: string): RepoReference[] {
|
||||
const repos: RepoReference[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
// Pattern to match GitHub URLs: github.com/owner/repo
|
||||
// Supports various formats:
|
||||
// - https://github.com/owner/repo
|
||||
// - http://github.com/owner/repo
|
||||
// - github.com/owner/repo
|
||||
// - [text](https://github.com/owner/repo)
|
||||
const patterns = [
|
||||
/https?:\/\/github\.com\/([a-zA-Z0-9_-]+)\/([a-zA-Z0-9_.-]+)/g,
|
||||
/github\.com\/([a-zA-Z0-9_-]+)\/([a-zA-Z0-9_.-]+)/g,
|
||||
];
|
||||
|
||||
for (const pattern of patterns) {
|
||||
let match;
|
||||
while ((match = pattern.exec(content)) !== null) {
|
||||
const owner = match[1];
|
||||
let repo = match[2];
|
||||
|
||||
// Clean up repo name (remove trailing slashes, .git, anchors, etc.)
|
||||
repo = repo.replace(/\/$/, '').replace(/\.git$/, '').split('/')[0].split('#')[0];
|
||||
|
||||
// Skip invalid repos
|
||||
if (!owner || !repo || repo.length === 0) continue;
|
||||
|
||||
// Skip common non-repo patterns
|
||||
if (
|
||||
repo === 'issues' ||
|
||||
repo === 'pulls' ||
|
||||
repo === 'blob' ||
|
||||
repo === 'tree' ||
|
||||
repo === 'raw' ||
|
||||
repo === 'releases'
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const key = `${owner}/${repo}`.toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
repos.push({
|
||||
owner,
|
||||
repo,
|
||||
url: `https://github.com/${owner}/${repo}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return repos;
|
||||
}
|
||||
|
||||
/**
|
||||
* Discover additional awesome lists from GitHub search
|
||||
*/
|
||||
async discoverAwesomeLists(): Promise<RepoReference[]> {
|
||||
const searchQueries = [
|
||||
'awesome-claude-skills in:name',
|
||||
'awesome-agent-skills in:name',
|
||||
'awesome claude skills in:description',
|
||||
'awesome ai skills in:description',
|
||||
];
|
||||
|
||||
const allRepos: RepoReference[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const query of searchQueries) {
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const response = await octokit.search.repos({
|
||||
q: query,
|
||||
sort: 'stars',
|
||||
order: 'desc',
|
||||
per_page: 30,
|
||||
});
|
||||
|
||||
for (const repo of response.data.items) {
|
||||
if (!repo.owner) continue;
|
||||
const key = `${repo.owner.login}/${repo.name}`.toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
allRepos.push({
|
||||
owner: repo.owner.login,
|
||||
repo: repo.name,
|
||||
url: repo.html_url,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`Search query failed: ${query}`, error);
|
||||
}
|
||||
}
|
||||
|
||||
return allRepos;
|
||||
}
|
||||
}
|
||||
|
||||
export function createAwesomeListCrawler(tokenManager?: TokenManager | string): AwesomeListCrawler {
|
||||
// Backward compatibility
|
||||
if (typeof tokenManager === 'string') {
|
||||
return new AwesomeListCrawler(new TokenManager([tokenManager]));
|
||||
}
|
||||
return new AwesomeListCrawler(tokenManager);
|
||||
}
|
||||
261
services/indexer/src/strategies/deep-scan.ts
Normal file
261
services/indexer/src/strategies/deep-scan.ts
Normal file
@@ -0,0 +1,261 @@
|
||||
import type { Octokit } from '@octokit/rest';
|
||||
import { TokenManager } from '../token-manager.js';import { OctokitPool } from '../octokit-pool.js';
|
||||
import type { SkillSource } from 'skillhub-core';
|
||||
import { INSTRUCTION_FILE_PATTERNS, type SourceFormat } from 'skillhub-core';
|
||||
|
||||
/**
|
||||
* Deep Scan Strategy
|
||||
* Uses Git Trees API to recursively scan entire repositories for SKILL.md files
|
||||
* Can discover skills that aren't found by code search
|
||||
*/
|
||||
export class DeepScanCrawler {
|
||||
private octokitPool: OctokitPool;
|
||||
private tokenManager: TokenManager;
|
||||
|
||||
constructor(tokenManager?: TokenManager) {
|
||||
this.tokenManager = tokenManager || TokenManager.getInstance();
|
||||
this.octokitPool = new OctokitPool(this.tokenManager);
|
||||
}
|
||||
|
||||
private async getOctokit(): Promise<Octokit> {
|
||||
return this.octokitPool.getBestInstance();
|
||||
}
|
||||
|
||||
private getCurrentToken(): string {
|
||||
return this.tokenManager.getBestToken();
|
||||
}
|
||||
|
||||
/**
|
||||
* Deep scan a repository for all SKILL.md files using Git Trees API
|
||||
* This is more thorough than code search as it scans the entire repo
|
||||
*/
|
||||
async scanRepository(owner: string, repo: string): Promise<SkillSource[]> {
|
||||
const skills: SkillSource[] = [];
|
||||
|
||||
try {
|
||||
// Get repository info for default branch
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const repoInfo = await octokit.repos.get({ owner, repo });
|
||||
this.octokitPool.updateStats(token, repoInfo.headers);
|
||||
|
||||
if (repoInfo.data.archived) {
|
||||
console.log(` Skipping archived repo: ${owner}/${repo}`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const defaultBranch = repoInfo.data.default_branch;
|
||||
|
||||
// Get the full tree recursively
|
||||
const tree = await octokit.git.getTree({
|
||||
owner,
|
||||
repo,
|
||||
tree_sha: defaultBranch,
|
||||
recursive: 'true',
|
||||
});
|
||||
this.octokitPool.updateStats(token, tree.headers);
|
||||
|
||||
// Find all instruction files (SKILL.md, .cursorrules, .windsurfrules, AGENTS.md, etc.)
|
||||
const instructionFiles = tree.data.tree.filter(
|
||||
(item) => {
|
||||
if (item.type !== 'blob' || !item.path) return false;
|
||||
return INSTRUCTION_FILE_PATTERNS.some(pattern => {
|
||||
if (pattern.rootOnly) return item.path === pattern.filename;
|
||||
if (pattern.pathFilter) return item.path!.includes(pattern.pathFilter) && (item.path!.endsWith('/' + pattern.filename) || item.path === pattern.filename);
|
||||
return item.path!.endsWith('/' + pattern.filename) || item.path === pattern.filename;
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
for (const file of instructionFiles) {
|
||||
if (!file.path) continue;
|
||||
const matchedPattern = INSTRUCTION_FILE_PATTERNS.find(pattern => {
|
||||
if (pattern.rootOnly) return file.path === pattern.filename;
|
||||
if (pattern.pathFilter) return file.path!.includes(pattern.pathFilter) && (file.path!.endsWith('/' + pattern.filename) || file.path === pattern.filename);
|
||||
return file.path!.endsWith('/' + pattern.filename) || file.path === pattern.filename;
|
||||
});
|
||||
if (!matchedPattern) continue;
|
||||
|
||||
const escapedFilename = matchedPattern.filename.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const skillPath = file.path.replace(new RegExp(`/?${escapedFilename}$`), '') || '.';
|
||||
|
||||
skills.push({
|
||||
owner,
|
||||
repo,
|
||||
path: skillPath,
|
||||
branch: defaultBranch,
|
||||
sourceFormat: matchedPattern.format,
|
||||
});
|
||||
}
|
||||
if (skills.length > 0) {
|
||||
console.log(` Found ${skills.length} skills in ${owner}/${repo}`);
|
||||
}
|
||||
|
||||
return skills;
|
||||
} catch (error) {
|
||||
if (this.isNotFoundError(error)) {
|
||||
return [];
|
||||
}
|
||||
if (this.isTruncatedError(error)) {
|
||||
// Tree is truncated (>100k files), fall back to directory listing
|
||||
console.log(` Repository ${owner}/${repo} is too large, using fallback scan`);
|
||||
return this.fallbackScan(owner, repo);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fallback scan for large repositories - scan known skill directories
|
||||
*/
|
||||
private async fallbackScan(owner: string, repo: string): Promise<SkillSource[]> {
|
||||
const skills: SkillSource[] = [];
|
||||
const knownPaths = ['skills', '.claude/skills', '.github/skills', '.codex/skills', ''];
|
||||
|
||||
for (const basePath of knownPaths) {
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
|
||||
const response = await octokit.repos.getContent({
|
||||
owner,
|
||||
repo,
|
||||
path: basePath || '.',
|
||||
});
|
||||
|
||||
this.octokitPool.updateStats(token, response.headers);
|
||||
|
||||
if (!Array.isArray(response.data)) continue;
|
||||
|
||||
// Check for SKILL.md at this level
|
||||
const hasSkillMd = response.data.some((item) => item.name === 'SKILL.md');
|
||||
if (hasSkillMd) {
|
||||
skills.push({
|
||||
owner,
|
||||
repo,
|
||||
path: basePath || '.',
|
||||
branch: 'main',
|
||||
sourceFormat: 'skill.md' as SourceFormat,
|
||||
});
|
||||
}
|
||||
|
||||
// Check subdirectories for SKILL.md
|
||||
const dirs = response.data.filter((item) => item.type === 'dir');
|
||||
for (const dir of dirs) {
|
||||
try {
|
||||
const subOctokit = await this.getOctokit();
|
||||
const subToken = this.getCurrentToken();
|
||||
const subDir = await subOctokit.repos.getContent({
|
||||
owner,
|
||||
repo,
|
||||
path: dir.path,
|
||||
});
|
||||
this.octokitPool.updateStats(subToken, subDir.headers);
|
||||
|
||||
if (Array.isArray(subDir.data)) {
|
||||
const hasSubSkillMd = subDir.data.some((item) => item.name === 'SKILL.md');
|
||||
if (hasSubSkillMd) {
|
||||
skills.push({
|
||||
owner,
|
||||
repo,
|
||||
path: dir.path,
|
||||
branch: 'main',
|
||||
sourceFormat: 'skill.md' as SourceFormat,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Skip inaccessible directories
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Path doesn't exist, continue
|
||||
}
|
||||
}
|
||||
|
||||
// Check for root-level instruction files (.cursorrules, .windsurfrules, AGENTS.md)
|
||||
const rootFiles = INSTRUCTION_FILE_PATTERNS.filter(p => p.format !== 'skill.md');
|
||||
for (const pattern of rootFiles) {
|
||||
try {
|
||||
const filePath = pattern.pathFilter ? `${pattern.pathFilter}${pattern.filename}` : pattern.filename;
|
||||
const fileContent = await this.getFileContent(owner, repo, filePath);
|
||||
if (fileContent && fileContent.length >= pattern.minContentLength) {
|
||||
skills.push({
|
||||
owner,
|
||||
repo,
|
||||
path: '.',
|
||||
branch: 'main',
|
||||
sourceFormat: pattern.format,
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// File doesn't exist
|
||||
}
|
||||
}
|
||||
|
||||
return skills;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan multiple repositories
|
||||
*/
|
||||
async scanRepositories(
|
||||
repos: Array<{ owner: string; repo: string }>
|
||||
): Promise<Map<string, SkillSource[]>> {
|
||||
const results = new Map<string, SkillSource[]>();
|
||||
|
||||
for (const { owner, repo } of repos) {
|
||||
try {
|
||||
console.log(`Deep scanning: ${owner}/${repo}`);
|
||||
const skills = await this.scanRepository(owner, repo);
|
||||
results.set(`${owner}/${repo}`, skills);
|
||||
} catch (error) {
|
||||
console.warn(` Failed to scan ${owner}/${repo}:`, error);
|
||||
results.set(`${owner}/${repo}`, []);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get file content from a repository
|
||||
*/
|
||||
async getFileContent(owner: string, repo: string, path: string): Promise<string | null> {
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const response = await octokit.repos.getContent({
|
||||
owner,
|
||||
repo,
|
||||
path,
|
||||
});
|
||||
this.octokitPool.updateStats(token, response.headers);
|
||||
|
||||
if ('content' in response.data && response.data.type === 'file') {
|
||||
return Buffer.from(response.data.content, 'base64').toString('utf-8');
|
||||
}
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private isNotFoundError(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Error && 'status' in error && (error as { status: number }).status === 404
|
||||
);
|
||||
}
|
||||
|
||||
private isTruncatedError(error: unknown): boolean {
|
||||
// Git Trees API returns truncated: true for repos >100k files
|
||||
return error instanceof Error && error.message.includes('truncated');
|
||||
}
|
||||
}
|
||||
|
||||
export function createDeepScanCrawler(tokenManager?: TokenManager | string): DeepScanCrawler {
|
||||
if (typeof tokenManager === 'string') {
|
||||
return new DeepScanCrawler(new TokenManager([tokenManager]));
|
||||
}
|
||||
return new DeepScanCrawler(tokenManager);
|
||||
}
|
||||
194
services/indexer/src/strategies/fork-network.ts
Normal file
194
services/indexer/src/strategies/fork-network.ts
Normal file
@@ -0,0 +1,194 @@
|
||||
import type { Octokit } from '@octokit/rest';
|
||||
import { TokenManager } from '../token-manager.js';import { OctokitPool } from '../octokit-pool.js';
|
||||
|
||||
export interface ForkInfo {
|
||||
owner: string;
|
||||
repo: string;
|
||||
stars: number;
|
||||
updatedAt: string;
|
||||
isArchived: boolean;
|
||||
defaultBranch: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fork Network Strategy
|
||||
* Discovers skills by traversing fork networks of known skill repositories
|
||||
* Forks often contain modified or additional skills
|
||||
*/
|
||||
export class ForkNetworkCrawler {
|
||||
private octokitPool: OctokitPool;
|
||||
private tokenManager: TokenManager;
|
||||
|
||||
constructor(tokenManager?: TokenManager) {
|
||||
this.tokenManager = tokenManager || TokenManager.getInstance();
|
||||
this.octokitPool = new OctokitPool(this.tokenManager);
|
||||
}
|
||||
|
||||
private async getOctokit(): Promise<Octokit> {
|
||||
return this.octokitPool.getBestInstance();
|
||||
}
|
||||
|
||||
private getCurrentToken(): string {
|
||||
return this.tokenManager.getBestToken();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all forks of a repository
|
||||
*/
|
||||
async getForks(owner: string, repo: string, maxPages = 10): Promise<ForkInfo[]> {
|
||||
const forks: ForkInfo[] = [];
|
||||
let page = 1;
|
||||
|
||||
while (page <= maxPages) {
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
|
||||
const response = await octokit.repos.listForks({
|
||||
owner,
|
||||
repo,
|
||||
sort: 'stargazers',
|
||||
per_page: 100,
|
||||
page,
|
||||
});
|
||||
|
||||
this.octokitPool.updateStats(token, response.headers);
|
||||
|
||||
if (response.data.length === 0) break;
|
||||
|
||||
for (const fork of response.data) {
|
||||
// Skip old/abandoned forks (not updated in 1 year)
|
||||
const updatedAt = fork.updated_at ? new Date(fork.updated_at) : new Date(0);
|
||||
const oneYearAgo = new Date();
|
||||
oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1);
|
||||
|
||||
if (updatedAt < oneYearAgo) continue;
|
||||
|
||||
forks.push({
|
||||
owner: fork.owner.login,
|
||||
repo: fork.name,
|
||||
stars: fork.stargazers_count ?? 0,
|
||||
updatedAt: fork.updated_at ?? new Date().toISOString(),
|
||||
isArchived: fork.archived ?? false,
|
||||
defaultBranch: fork.default_branch ?? 'main',
|
||||
});
|
||||
}
|
||||
|
||||
if (response.data.length < 100) break;
|
||||
page++;
|
||||
} catch (error) {
|
||||
console.warn(`Failed to get forks page ${page} for ${owner}/${repo}:`, error);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return forks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get forks for multiple seed repositories
|
||||
*/
|
||||
async discoverFromSeedRepos(
|
||||
seedRepos: Array<{ owner: string; repo: string }>
|
||||
): Promise<ForkInfo[]> {
|
||||
const allForks: ForkInfo[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const seed of seedRepos) {
|
||||
try {
|
||||
console.log(`Fetching forks of ${seed.owner}/${seed.repo}...`);
|
||||
const forks = await this.getForks(seed.owner, seed.repo);
|
||||
|
||||
for (const fork of forks) {
|
||||
const key = `${fork.owner}/${fork.repo}`.toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
allForks.push(fork);
|
||||
}
|
||||
|
||||
console.log(` Found ${forks.length} active forks`);
|
||||
} catch (error) {
|
||||
console.warn(`Failed to get forks for ${seed.owner}/${seed.repo}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Total unique forks discovered: ${allForks.length}`);
|
||||
return allForks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get parent/source repository (for finding the original if we have a fork)
|
||||
*/
|
||||
async getParentRepo(owner: string, repo: string): Promise<{ owner: string; repo: string } | null> {
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const response = await octokit.repos.get({ owner, repo });
|
||||
this.octokitPool.updateStats(token, response.headers);
|
||||
|
||||
if (response.data.fork && response.data.parent) {
|
||||
return {
|
||||
owner: response.data.parent.owner.login,
|
||||
repo: response.data.parent.name,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all repositories in the same network (parent + siblings)
|
||||
*/
|
||||
async getNetworkRepos(owner: string, repo: string): Promise<ForkInfo[]> {
|
||||
const repos: ForkInfo[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
// Get parent if this is a fork
|
||||
const parent = await this.getParentRepo(owner, repo);
|
||||
const rootOwner = parent?.owner || owner;
|
||||
const rootRepo = parent?.repo || repo;
|
||||
|
||||
// Add root repo
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const rootInfo = await octokit.repos.get({ owner: rootOwner, repo: rootRepo });
|
||||
this.octokitPool.updateStats(token, rootInfo.headers);
|
||||
|
||||
const rootKey = `${rootOwner}/${rootRepo}`.toLowerCase();
|
||||
seen.add(rootKey);
|
||||
|
||||
repos.push({
|
||||
owner: rootOwner,
|
||||
repo: rootRepo,
|
||||
stars: rootInfo.data.stargazers_count,
|
||||
updatedAt: rootInfo.data.updated_at,
|
||||
isArchived: rootInfo.data.archived,
|
||||
defaultBranch: rootInfo.data.default_branch,
|
||||
});
|
||||
} catch {
|
||||
// Root not accessible
|
||||
}
|
||||
|
||||
// Get all forks of root
|
||||
const forks = await this.getForks(rootOwner, rootRepo);
|
||||
for (const fork of forks) {
|
||||
const key = `${fork.owner}/${fork.repo}`.toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
repos.push(fork);
|
||||
}
|
||||
|
||||
return repos;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export function createForkNetworkCrawler(tokenManager?: TokenManager | string): ForkNetworkCrawler {
|
||||
if (typeof tokenManager === 'string') {
|
||||
return new ForkNetworkCrawler(new TokenManager([tokenManager]));
|
||||
}
|
||||
return new ForkNetworkCrawler(tokenManager);
|
||||
}
|
||||
292
services/indexer/src/strategies/index.ts
Normal file
292
services/indexer/src/strategies/index.ts
Normal file
@@ -0,0 +1,292 @@
|
||||
/**
|
||||
* Strategy Orchestrator
|
||||
* Coordinates multiple discovery strategies for comprehensive skill discovery
|
||||
*/
|
||||
|
||||
import { createAwesomeListCrawler } from './awesome-list.js';
|
||||
import { createTopicSearchCrawler } from './topic-search.js';
|
||||
import { createDeepScanCrawler } from './deep-scan.js';
|
||||
import { createForkNetworkCrawler } from './fork-network.js';
|
||||
import { TokenManager } from '../token-manager.js';
|
||||
import type { SkillSource } from 'skillhub-core';
|
||||
|
||||
export interface DiscoveryResult {
|
||||
source: string;
|
||||
repos: Array<{
|
||||
owner: string;
|
||||
repo: string;
|
||||
stars?: number;
|
||||
discoveredVia: string;
|
||||
}>;
|
||||
skills: SkillSource[];
|
||||
}
|
||||
|
||||
export interface DiscoveryStats {
|
||||
totalReposDiscovered: number;
|
||||
totalSkillsFound: number;
|
||||
byStrategy: {
|
||||
[key: string]: {
|
||||
repos: number;
|
||||
skills: number;
|
||||
};
|
||||
};
|
||||
duration: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strategy Orchestrator - coordinates all discovery strategies
|
||||
*/
|
||||
export class StrategyOrchestrator {
|
||||
private awesomeCrawler: ReturnType<typeof createAwesomeListCrawler>;
|
||||
private topicCrawler: ReturnType<typeof createTopicSearchCrawler>;
|
||||
private deepScanCrawler: ReturnType<typeof createDeepScanCrawler>;
|
||||
private forkCrawler: ReturnType<typeof createForkNetworkCrawler>;
|
||||
private tokenManager: TokenManager;
|
||||
|
||||
constructor(tokenManager?: TokenManager) {
|
||||
this.tokenManager = tokenManager || TokenManager.getInstance();
|
||||
this.awesomeCrawler = createAwesomeListCrawler(this.tokenManager);
|
||||
this.topicCrawler = createTopicSearchCrawler(this.tokenManager);
|
||||
this.deepScanCrawler = createDeepScanCrawler(this.tokenManager);
|
||||
this.forkCrawler = createForkNetworkCrawler(this.tokenManager);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run awesome list discovery
|
||||
*/
|
||||
async runAwesomeListStrategy(): Promise<DiscoveryResult> {
|
||||
console.log('\n=== Running Awesome List Strategy ===');
|
||||
const startTime = Date.now();
|
||||
|
||||
const listResults = await this.awesomeCrawler.crawlAllLists();
|
||||
|
||||
const repos: DiscoveryResult['repos'] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const repoRefs of listResults.values()) {
|
||||
for (const ref of repoRefs) {
|
||||
const key = `${ref.owner}/${ref.repo}`.toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
repos.push({
|
||||
owner: ref.owner,
|
||||
repo: ref.repo,
|
||||
discoveredVia: 'awesome-list',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Awesome list strategy completed in ${Date.now() - startTime}ms`);
|
||||
console.log(`Discovered ${repos.length} repositories from ${listResults.size} lists`);
|
||||
|
||||
return {
|
||||
source: 'awesome-list',
|
||||
repos,
|
||||
skills: [],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run topic search discovery
|
||||
*/
|
||||
async runTopicSearchStrategy(): Promise<DiscoveryResult> {
|
||||
console.log('\n=== Running Topic Search Strategy ===');
|
||||
const startTime = Date.now();
|
||||
|
||||
const repoResults = await this.topicCrawler.discoverAll();
|
||||
|
||||
const repos = repoResults.map((r) => ({
|
||||
owner: r.owner,
|
||||
repo: r.repo,
|
||||
stars: r.stars,
|
||||
discoveredVia: 'topic-search',
|
||||
}));
|
||||
|
||||
console.log(`Topic search strategy completed in ${Date.now() - startTime}ms`);
|
||||
console.log(`Discovered ${repos.length} repositories`);
|
||||
|
||||
return {
|
||||
source: 'topic-search',
|
||||
repos,
|
||||
skills: [],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run deep scan on discovered repositories
|
||||
*/
|
||||
async runDeepScanStrategy(
|
||||
repos: Array<{ owner: string; repo: string }>
|
||||
): Promise<DiscoveryResult> {
|
||||
console.log('\n=== Running Deep Scan Strategy ===');
|
||||
const startTime = Date.now();
|
||||
|
||||
const scanResults = await this.deepScanCrawler.scanRepositories(repos);
|
||||
|
||||
const allSkills: SkillSource[] = [];
|
||||
for (const skills of scanResults.values()) {
|
||||
allSkills.push(...skills);
|
||||
}
|
||||
|
||||
console.log(`Deep scan strategy completed in ${Date.now() - startTime}ms`);
|
||||
console.log(`Found ${allSkills.length} skills in ${repos.length} repositories`);
|
||||
|
||||
return {
|
||||
source: 'deep-scan',
|
||||
repos: [],
|
||||
skills: allSkills,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run fork network discovery
|
||||
*/
|
||||
async runForkNetworkStrategy(
|
||||
seedRepos: Array<{ owner: string; repo: string }>
|
||||
): Promise<DiscoveryResult> {
|
||||
console.log('\n=== Running Fork Network Strategy ===');
|
||||
const startTime = Date.now();
|
||||
|
||||
const forks = await this.forkCrawler.discoverFromSeedRepos(seedRepos);
|
||||
|
||||
const repos = forks
|
||||
.filter((f) => !f.isArchived)
|
||||
.map((f) => ({
|
||||
owner: f.owner,
|
||||
repo: f.repo,
|
||||
stars: f.stars,
|
||||
discoveredVia: 'fork-network',
|
||||
}));
|
||||
|
||||
console.log(`Fork network strategy completed in ${Date.now() - startTime}ms`);
|
||||
console.log(`Discovered ${repos.length} active forks`);
|
||||
|
||||
return {
|
||||
source: 'fork-network',
|
||||
repos,
|
||||
skills: [],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run all strategies and return combined results
|
||||
*/
|
||||
async runAllStrategies(): Promise<{
|
||||
repos: Array<{
|
||||
owner: string;
|
||||
repo: string;
|
||||
stars?: number;
|
||||
discoveredVia: string;
|
||||
}>;
|
||||
stats: DiscoveryStats;
|
||||
}> {
|
||||
const startTime = Date.now();
|
||||
const stats: DiscoveryStats = {
|
||||
totalReposDiscovered: 0,
|
||||
totalSkillsFound: 0,
|
||||
byStrategy: {},
|
||||
duration: 0,
|
||||
};
|
||||
|
||||
const allRepos: Map<string, {
|
||||
owner: string;
|
||||
repo: string;
|
||||
stars?: number;
|
||||
discoveredVia: string;
|
||||
}> = new Map();
|
||||
|
||||
// 1. Awesome lists (fast, high yield)
|
||||
try {
|
||||
const awesomeResult = await this.runAwesomeListStrategy();
|
||||
stats.byStrategy['awesome-list'] = {
|
||||
repos: awesomeResult.repos.length,
|
||||
skills: 0,
|
||||
};
|
||||
for (const repo of awesomeResult.repos) {
|
||||
const key = `${repo.owner}/${repo.repo}`.toLowerCase();
|
||||
if (!allRepos.has(key)) {
|
||||
allRepos.set(key, repo);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Awesome list strategy failed:', error);
|
||||
}
|
||||
|
||||
// 2. Topic search (medium speed, medium yield)
|
||||
try {
|
||||
const topicResult = await this.runTopicSearchStrategy();
|
||||
stats.byStrategy['topic-search'] = {
|
||||
repos: topicResult.repos.length,
|
||||
skills: 0,
|
||||
};
|
||||
for (const repo of topicResult.repos) {
|
||||
const key = `${repo.owner}/${repo.repo}`.toLowerCase();
|
||||
if (!allRepos.has(key)) {
|
||||
allRepos.set(key, repo);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Topic search strategy failed:', error);
|
||||
}
|
||||
|
||||
// 3. Fork network (for high-star repos from above)
|
||||
try {
|
||||
const highStarRepos = Array.from(allRepos.values())
|
||||
.filter((r) => (r.stars || 0) >= 10)
|
||||
.slice(0, 50); // Top 50 repos by stars
|
||||
|
||||
if (highStarRepos.length > 0) {
|
||||
const forkResult = await this.runForkNetworkStrategy(
|
||||
highStarRepos.map((r) => ({ owner: r.owner, repo: r.repo }))
|
||||
);
|
||||
stats.byStrategy['fork-network'] = {
|
||||
repos: forkResult.repos.length,
|
||||
skills: 0,
|
||||
};
|
||||
for (const repo of forkResult.repos) {
|
||||
const key = `${repo.owner}/${repo.repo}`.toLowerCase();
|
||||
if (!allRepos.has(key)) {
|
||||
allRepos.set(key, repo);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Fork network strategy failed:', error);
|
||||
}
|
||||
|
||||
stats.totalReposDiscovered = allRepos.size;
|
||||
stats.duration = Date.now() - startTime;
|
||||
|
||||
console.log('\n=== Discovery Summary ===');
|
||||
console.log(`Total unique repositories: ${allRepos.size}`);
|
||||
console.log(`Duration: ${(stats.duration / 1000).toFixed(1)}s`);
|
||||
console.log('By strategy:');
|
||||
for (const [strategy, data] of Object.entries(stats.byStrategy)) {
|
||||
console.log(` ${strategy}: ${data.repos} repos`);
|
||||
}
|
||||
|
||||
return {
|
||||
repos: Array.from(allRepos.values()),
|
||||
stats,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export function createStrategyOrchestrator(tokenManager?: TokenManager | string): StrategyOrchestrator {
|
||||
if (typeof tokenManager === 'string') {
|
||||
return new StrategyOrchestrator(new TokenManager([tokenManager]));
|
||||
}
|
||||
return new StrategyOrchestrator(tokenManager);
|
||||
}
|
||||
|
||||
// Re-export strategy creators
|
||||
export { createAwesomeListCrawler } from './awesome-list.js';
|
||||
export { createTopicSearchCrawler } from './topic-search.js';
|
||||
export { createDeepScanCrawler } from './deep-scan.js';
|
||||
export { createForkNetworkCrawler } from './fork-network.js';
|
||||
|
||||
// Re-export types
|
||||
export type { RepoReference } from './awesome-list.js';
|
||||
export type { RepoResult } from './topic-search.js';
|
||||
export type { ForkInfo } from './fork-network.js';
|
||||
314
services/indexer/src/strategies/topic-search.ts
Normal file
314
services/indexer/src/strategies/topic-search.ts
Normal file
@@ -0,0 +1,314 @@
|
||||
import type { Octokit } from '@octokit/rest';
|
||||
import { TokenManager } from '../token-manager.js';import { OctokitPool } from '../octokit-pool.js';
|
||||
|
||||
export interface RepoResult {
|
||||
owner: string;
|
||||
repo: string;
|
||||
stars: number;
|
||||
forks: number;
|
||||
description: string | null;
|
||||
topics: string[];
|
||||
defaultBranch: string;
|
||||
isArchived: boolean;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Topics that indicate a repository might contain skills
|
||||
* Includes both compound topics (claude-skills) and simple topics (skill)
|
||||
*/
|
||||
const SKILL_TOPICS = [
|
||||
// Compound skill topics
|
||||
'claude-skills',
|
||||
'agent-skills',
|
||||
'ai-skills',
|
||||
'claude-code',
|
||||
'codex-skills',
|
||||
'copilot-skills',
|
||||
'skill-md',
|
||||
'anthropic-skills',
|
||||
'claude-code-skills',
|
||||
'ai-agent-skills',
|
||||
// Simple skill topics (for repos like openclaw/skills with topic "skill")
|
||||
'skill',
|
||||
'skills',
|
||||
// Platform-specific
|
||||
'clawdhub',
|
||||
'clawdbot',
|
||||
'skillhub',
|
||||
// AI agent related
|
||||
'ai-agent',
|
||||
'claude-agent',
|
||||
'anthropic',
|
||||
'llm-skills',
|
||||
'mcp-skills',
|
||||
// Cursor-specific
|
||||
'cursor-rules',
|
||||
'cursorrules',
|
||||
// Windsurf-specific
|
||||
'windsurf-rules',
|
||||
'windsurfrules',
|
||||
// Copilot-specific
|
||||
'copilot-instructions',
|
||||
'github-copilot',
|
||||
// Codex-specific
|
||||
'codex-agent',
|
||||
'openai-codex',
|
||||
];
|
||||
|
||||
/**
|
||||
* Search queries for repository descriptions and READMEs
|
||||
*/
|
||||
const REPO_SEARCH_QUERIES = [
|
||||
'SKILL.md in:readme',
|
||||
'claude skills in:description',
|
||||
'agent skills in:description',
|
||||
'"agent skill" in:readme',
|
||||
'claude code skill in:readme',
|
||||
'codex skill in:readme',
|
||||
'skill.md file in:readme',
|
||||
// Additional queries for platforms like clawdhub
|
||||
'clawdhub in:description',
|
||||
'clawdbot in:description',
|
||||
'claude code skills in:description',
|
||||
'agent skill in:name',
|
||||
'skills archive in:description',
|
||||
'anthropic skills in:description',
|
||||
'.cursorrules in:readme',
|
||||
'cursor rules in:description',
|
||||
'AGENTS.md in:readme',
|
||||
'codex agents in:description',
|
||||
'copilot-instructions in:readme',
|
||||
'windsurf rules in:description',
|
||||
'.windsurfrules in:readme',
|
||||
];
|
||||
|
||||
/**
|
||||
* Topic Search Strategy
|
||||
* Discovers repositories using GitHub's topic and description search
|
||||
*/
|
||||
export class TopicSearchCrawler {
|
||||
private octokitPool: OctokitPool;
|
||||
private tokenManager: TokenManager;
|
||||
|
||||
constructor(tokenManager?: TokenManager) {
|
||||
this.tokenManager = tokenManager || TokenManager.getInstance();
|
||||
this.octokitPool = new OctokitPool(this.tokenManager);
|
||||
}
|
||||
|
||||
private async getOctokit(): Promise<Octokit> {
|
||||
return this.octokitPool.getBestInstance();
|
||||
}
|
||||
|
||||
private getCurrentToken(): string {
|
||||
return this.tokenManager.getBestToken();
|
||||
}
|
||||
|
||||
/**
|
||||
* Search by all known skill-related topics
|
||||
*/
|
||||
async searchByTopics(): Promise<RepoResult[]> {
|
||||
const allRepos: RepoResult[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
console.log(`Searching by ${SKILL_TOPICS.length} skill-related topics...`);
|
||||
|
||||
for (const topic of SKILL_TOPICS) {
|
||||
try {
|
||||
console.log(` Searching topic: ${topic}`);
|
||||
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const response = await octokit.search.repos({
|
||||
q: `topic:${topic}`,
|
||||
sort: 'stars',
|
||||
order: 'desc',
|
||||
per_page: 100,
|
||||
});
|
||||
|
||||
this.octokitPool.updateStats(token, response.headers);
|
||||
console.log(` Found ${response.data.total_count} repos`);
|
||||
|
||||
for (const repo of response.data.items) {
|
||||
if (!repo.owner) continue;
|
||||
const key = `${repo.owner.login}/${repo.name}`.toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
allRepos.push({
|
||||
owner: repo.owner.login,
|
||||
repo: repo.name,
|
||||
stars: repo.stargazers_count,
|
||||
forks: repo.forks_count,
|
||||
description: repo.description,
|
||||
topics: repo.topics || [],
|
||||
defaultBranch: repo.default_branch,
|
||||
isArchived: repo.archived,
|
||||
updatedAt: repo.updated_at,
|
||||
});
|
||||
}
|
||||
|
||||
// Paginate if more results
|
||||
if (response.data.total_count > 100) {
|
||||
const additionalRepos = await this.paginateSearch(`topic:${topic}`, seen, 5);
|
||||
allRepos.push(...additionalRepos);
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(` Topic search failed for ${topic}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Topic search found ${allRepos.length} unique repositories`);
|
||||
return allRepos;
|
||||
}
|
||||
|
||||
/**
|
||||
* Search by description and README content
|
||||
*/
|
||||
async searchByDescription(): Promise<RepoResult[]> {
|
||||
const allRepos: RepoResult[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
console.log(`Searching by ${REPO_SEARCH_QUERIES.length} description queries...`);
|
||||
|
||||
for (const query of REPO_SEARCH_QUERIES) {
|
||||
try {
|
||||
console.log(` Query: ${query}`);
|
||||
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const response = await octokit.search.repos({
|
||||
q: query,
|
||||
sort: 'stars',
|
||||
order: 'desc',
|
||||
per_page: 100,
|
||||
});
|
||||
|
||||
this.octokitPool.updateStats(token, response.headers);
|
||||
console.log(` Found ${response.data.total_count} repos`);
|
||||
|
||||
for (const repo of response.data.items) {
|
||||
if (!repo.owner) continue;
|
||||
const key = `${repo.owner.login}/${repo.name}`.toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
allRepos.push({
|
||||
owner: repo.owner.login,
|
||||
repo: repo.name,
|
||||
stars: repo.stargazers_count,
|
||||
forks: repo.forks_count,
|
||||
description: repo.description,
|
||||
topics: repo.topics || [],
|
||||
defaultBranch: repo.default_branch,
|
||||
isArchived: repo.archived,
|
||||
updatedAt: repo.updated_at,
|
||||
});
|
||||
}
|
||||
|
||||
// Paginate if more results
|
||||
if (response.data.total_count > 100) {
|
||||
const additionalRepos = await this.paginateSearch(query, seen, 5);
|
||||
allRepos.push(...additionalRepos);
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(` Description search failed for "${query}":`, error);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Description search found ${allRepos.length} unique repositories`);
|
||||
return allRepos;
|
||||
}
|
||||
|
||||
/**
|
||||
* Paginate through search results
|
||||
*/
|
||||
private async paginateSearch(
|
||||
query: string,
|
||||
seen: Set<string>,
|
||||
maxPages: number
|
||||
): Promise<RepoResult[]> {
|
||||
const repos: RepoResult[] = [];
|
||||
|
||||
for (let page = 2; page <= maxPages; page++) {
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
|
||||
const response = await octokit.search.repos({
|
||||
q: query,
|
||||
sort: 'stars',
|
||||
order: 'desc',
|
||||
per_page: 100,
|
||||
page,
|
||||
});
|
||||
|
||||
this.octokitPool.updateStats(token, response.headers);
|
||||
|
||||
if (response.data.items.length === 0) break;
|
||||
|
||||
for (const repo of response.data.items) {
|
||||
if (!repo.owner) continue;
|
||||
const key = `${repo.owner.login}/${repo.name}`.toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
repos.push({
|
||||
owner: repo.owner.login,
|
||||
repo: repo.name,
|
||||
stars: repo.stargazers_count,
|
||||
forks: repo.forks_count,
|
||||
description: repo.description,
|
||||
topics: repo.topics || [],
|
||||
defaultBranch: repo.default_branch,
|
||||
isArchived: repo.archived,
|
||||
updatedAt: repo.updated_at,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
// Stop on 422 (beyond 1000 results)
|
||||
if (this.isBeyondResultsLimit(error)) break;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
return repos;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run all topic and description searches
|
||||
*/
|
||||
async discoverAll(): Promise<RepoResult[]> {
|
||||
const topicRepos = await this.searchByTopics();
|
||||
const descRepos = await this.searchByDescription();
|
||||
|
||||
// Merge and deduplicate
|
||||
const seen = new Set<string>();
|
||||
const allRepos: RepoResult[] = [];
|
||||
|
||||
for (const repo of [...topicRepos, ...descRepos]) {
|
||||
const key = `${repo.owner}/${repo.repo}`.toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
allRepos.push(repo);
|
||||
}
|
||||
|
||||
console.log(`Total unique repositories discovered: ${allRepos.length}`);
|
||||
return allRepos;
|
||||
}
|
||||
|
||||
private isBeyondResultsLimit(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Error &&
|
||||
'status' in error &&
|
||||
(error as { status: number }).status === 422
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function createTopicSearchCrawler(tokenManager?: TokenManager | string): TopicSearchCrawler {
|
||||
if (typeof tokenManager === 'string') {
|
||||
return new TopicSearchCrawler(new TokenManager([tokenManager]));
|
||||
}
|
||||
return new TopicSearchCrawler(tokenManager);
|
||||
}
|
||||
Reference in New Issue
Block a user