atlatl/skillhub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(indexer): prevent token exhaustion and reserve 20% budget for website users

Browse Source 
- Lower exhaustion threshold from <10 to <2 (stop wasting remaining tokens)
- Refresh ALL tokens after wait instead of only one
- Fix token/Octokit mismatch by returning token from getBestInstance()
- Add budget checking (20% reserve) to deep-scan, discover-repos, awesome-lists,
  and full-enhanced commands
- Add rate limit error handling with retry in DeepScanCrawler
- Sync translation and query changes

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
airano
2026-02-14 18:29:39 +03:30
parent e8bf89d311
commit 444a6668d0
11 changed files with 101 additions and 364 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
apps/web/messages/en.json
View File

@@ -33,7 +33,7 @@
"docs": "Docs", "docs": "Docs",
"about": "About", "about": "About",
"github": "GitHub", "github": "GitHub",
"sourceCode": "Source Code (Coming Soon)", "sourceCode": "Source Code",
"sourceCodeShort": "Source Code", "sourceCodeShort": "Source Code",
"switchLanguage": "Switch to Farsi" "switchLanguage": "Switch to Farsi"
}, },
@@ -203,7 +203,7 @@
"attribution": "Attribution", "attribution": "Attribution",
"manageSkills": "Manage Skills" "manageSkills": "Manage Skills"
}, },
"sourceCode": "Source Code (Coming Soon)", "sourceCode": "Source Code",
"copyright": "© {year} SkillHub. Open source under MIT license." "copyright": "© {year} SkillHub. Open source under MIT license."
}, },
"about": { "about": {

4
apps/web/messages/fa.json
View File

@@ -33,7 +33,7 @@
"docs": "مستندات", "docs": "مستندات",
"about": "درباره ما", "about": "درباره ما",
"github": "گیت‌هاب", "github": "گیت‌هاب",
"sourceCode": "کد منبع (به زودی)", "sourceCode": "کد منبع",
"sourceCodeShort": "کد منبع", "sourceCodeShort": "کد منبع",
"switchLanguage": "تغییر به انگلیسی" "switchLanguage": "تغییر به انگلیسی"
}, },
@@ -203,7 +203,7 @@
"attribution": "تقدیر و تشکر", "attribution": "تقدیر و تشکر",
"manageSkills": "مدیریت مهارت‌ها" "manageSkills": "مدیریت مهارت‌ها"
}, },
"sourceCode": "کد منبع (به زودی)", "sourceCode": "کد منبع",
"copyright": "© {year} SkillHub. کد متن‌باز تحت مجوز MIT." "copyright": "© {year} SkillHub. کد متن‌باز تحت مجوز MIT."
}, },
"about": { "about": {

295
packages/db/src/queries.ts
View File

@@ -15,9 +15,6 @@ import {
addRequests, addRequests,
emailSubscriptions, emailSubscriptions,
} from './schema.js'; } from './schema.js';
// #region private-only
import { outreachEmails } from './schema.js';
// #endregion private-only
import type * as schema from './schema.js'; import type * as schema from './schema.js';
@@ -2027,295 +2024,3 @@ export const emailSubscriptionQueries = {
}, },
}; };
// #region private-only
/**
* Outreach email queries
*/
export const outreachQueries = {
/**
* Upsert an outreach record for a GitHub owner
*/
upsert: async (
db: DB,
data: {
githubUsername: string;
email?: string | null;
emailSource?: string | null;
repoCount?: number;
skillCount?: number;
totalStars?: number;
status?: string;
templateData?: typeof outreachEmails.$inferInsert['templateData'];
}
) => {
const result = await db
.insert(outreachEmails)
.values({
id: crypto.randomUUID(),
githubUsername: data.githubUsername,
email: data.email ?? undefined,
emailSource: data.emailSource ?? undefined,
repoCount: data.repoCount ?? 0,
skillCount: data.skillCount ?? 0,
totalStars: data.totalStars ?? 0,
status: data.status ?? (data.email ? 'pending' : 'no_email'),
templateData: data.templateData ?? undefined,
})
.onConflictDoUpdate({
target: outreachEmails.githubUsername,
set: {
...(data.email !== undefined ? { email: data.email } : {}),
...(data.emailSource !== undefined ? { emailSource: data.emailSource } : {}),
...(data.repoCount !== undefined ? { repoCount: data.repoCount } : {}),
...(data.skillCount !== undefined ? { skillCount: data.skillCount } : {}),
...(data.totalStars !== undefined ? { totalStars: data.totalStars } : {}),
...(data.status !== undefined ? { status: data.status } : {}),
...(data.templateData !== undefined ? { templateData: data.templateData } : {}),
updatedAt: new Date(),
},
})
.returning();
return result[0];
},
/**
* Get pending outreach emails (with email, not yet sent)
* Three-tier priority:
* Tier 0: owners with 1..priorityMaxStars (sweet spot, most likely to engage)
* Tier 1: owners with >priorityMaxStars (big orgs, less likely to engage)
* Tier 2: owners with 0 stars (least valuable, sent last)
* Within each tier, sorted by stars descending
*/
getPending: async (db: DB, limit: number = 70, priorityMaxStars: number = 10000) => {
return db
.select()
.from(outreachEmails)
.where(
and(
eq(outreachEmails.status, 'pending'),
sql`${outreachEmails.email} IS NOT NULL`
)
)
.orderBy(
sql`CASE
WHEN ${outreachEmails.totalStars} BETWEEN 1 AND ${priorityMaxStars} THEN 0
WHEN ${outreachEmails.totalStars} > ${priorityMaxStars} THEN 1
ELSE 2
END`,
desc(outreachEmails.totalStars)
)
.limit(limit);
},
/**
* Mark an outreach email as sent
*/
markSent: async (db: DB, githubUsername: string, resendEmailId?: string) => {
return db
.update(outreachEmails)
.set({
status: 'sent',
sentAt: new Date(),
resendEmailId: resendEmailId ?? undefined,
})
.where(eq(outreachEmails.githubUsername, githubUsername));
},
/**
* Reset falsely sent records back to pending (where resend_email_id is null)
* Used to fix records that were marked sent but never actually sent
*/
resetFalselySent: async (db: DB) => {
const result = await db
.update(outreachEmails)
.set({
status: 'pending',
sentAt: null,
})
.where(
and(
eq(outreachEmails.status, 'sent'),
sql`${outreachEmails.resendEmailId} IS NULL`
)
)
.returning({ githubUsername: outreachEmails.githubUsername });
return result.length;
},
/**
* Reset failed records back to pending (e.g. after rate limiting)
* Optionally filter by error message pattern
*/
resetFailed: async (db: DB, errorPattern?: string) => {
const conditions = [eq(outreachEmails.status, 'failed')];
if (errorPattern) {
conditions.push(like(outreachEmails.errorMessage, `%${errorPattern}%`));
}
const result = await db
.update(outreachEmails)
.set({
status: 'pending',
errorMessage: null,
})
.where(and(...conditions))
.returning({ githubUsername: outreachEmails.githubUsername });
return result.length;
},
/**
* Mark an outreach email as failed
*/
markFailed: async (db: DB, githubUsername: string, error: string) => {
return db
.update(outreachEmails)
.set({
status: 'failed',
errorMessage: error,
})
.where(eq(outreachEmails.githubUsername, githubUsername));
},
/**
* Mark an outreach email as unsubscribed
*/
markUnsubscribed: async (db: DB, githubUsername: string) => {
return db
.update(outreachEmails)
.set({ status: 'unsubscribed' })
.where(eq(outreachEmails.githubUsername, githubUsername));
},
/**
* Get outreach stats by status
*/
getStats: async (db: DB) => {
return db
.select({
status: outreachEmails.status,
count: sql<number>`count(*)::int`,
})
.from(outreachEmails)
.groupBy(outreachEmails.status);
},
/**
* Get star distribution for pending outreach emails (with email)
* Useful for understanding the priority tiers
*/
getStarDistribution: async (db: DB) => {
return db.execute(sql`
SELECT
CASE
WHEN total_stars = 0 THEN '0'
WHEN total_stars BETWEEN 1 AND 100 THEN '1-100'
WHEN total_stars BETWEEN 101 AND 500 THEN '101-500'
WHEN total_stars BETWEEN 501 AND 1000 THEN '501-1K'
WHEN total_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
WHEN total_stars BETWEEN 5001 AND 10000 THEN '5K-10K'
WHEN total_stars BETWEEN 10001 AND 50000 THEN '10K-50K'
ELSE '50K+'
END as star_range,
COUNT(*) as total,
COUNT(*) FILTER (WHERE status = 'pending' AND email IS NOT NULL) as sendable,
COUNT(*) FILTER (WHERE status = 'sent') as sent,
COUNT(*) FILTER (WHERE status = 'no_email') as no_email
FROM outreach_emails
GROUP BY
CASE
WHEN total_stars = 0 THEN '0'
WHEN total_stars BETWEEN 1 AND 100 THEN '1-100'
WHEN total_stars BETWEEN 101 AND 500 THEN '101-500'
WHEN total_stars BETWEEN 501 AND 1000 THEN '501-1K'
WHEN total_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
WHEN total_stars BETWEEN 5001 AND 10000 THEN '5K-10K'
WHEN total_stars BETWEEN 10001 AND 50000 THEN '10K-50K'
ELSE '50K+'
END
ORDER BY MIN(total_stars)
`);
},
/**
* Populate outreach_emails from skills table (aggregated by owner)
* Only inserts new owners that don't already exist in outreach_emails
*/
populateFromSkills: async (_db: DB) => {
const rawClient = getRawClientLocal();
const result = await rawClient`
INSERT INTO outreach_emails (id, github_username, repo_count, skill_count, total_stars, status, template_data, created_at, updated_at)
SELECT
gen_random_uuid()::TEXT,
s.github_owner,
COUNT(DISTINCT s.github_repo)::INTEGER,
COUNT(*)::INTEGER,
COALESCE(MAX(s.github_stars), 0),
'pending',
jsonb_build_object(
'repos', (
SELECT jsonb_agg(repo_data ORDER BY repo_data->>'stars' DESC)
FROM (
SELECT jsonb_build_object(
'name', sub.github_repo,
'skillCount', sub.cnt,
'stars', sub.max_stars,
'topSkills', sub.top_skills
) as repo_data
FROM (
SELECT
s2.github_repo,
COUNT(*)::INTEGER as cnt,
MAX(s2.github_stars) as max_stars,
jsonb_agg(
jsonb_build_object('id', s2.id, 'name', s2.name, 'description', LEFT(s2.description, 100))
ORDER BY s2.github_stars DESC
) FILTER (WHERE s2.rn <= 3) as top_skills
FROM (
SELECT *, ROW_NUMBER() OVER (PARTITION BY github_owner, github_repo ORDER BY github_stars DESC) as rn
FROM skills
WHERE github_owner = s.github_owner AND is_blocked = false
) s2
GROUP BY s2.github_repo
) sub
) repo_sub
)
),
NOW(),
NOW()
FROM skills s
WHERE s.is_blocked = false
GROUP BY s.github_owner
ON CONFLICT (github_username) DO UPDATE SET
repo_count = EXCLUDED.repo_count,
skill_count = EXCLUDED.skill_count,
total_stars = EXCLUDED.total_stars,
template_data = EXCLUDED.template_data,
updated_at = NOW()
WHERE outreach_emails.status NOT IN ('sent', 'unsubscribed')
`;
return result.count;
},
/**
* Get by github username
*/
getByUsername: async (db: DB, githubUsername: string) => {
const result = await db
.select()
.from(outreachEmails)
.where(eq(outreachEmails.githubUsername, githubUsername))
.limit(1);
return result[0] ?? null;
},
/**
* Get total skill count (for dynamic numbers in emails)
*/
getTotalSkillCount: async (db: DB): Promise<number> => {
const result = await db
.select({ count: sql<number>`count(*)::int` })
.from(skills)
.where(eq(skills.isBlocked, false));
return result[0]?.count ?? 0;
},
};
// #endregion private-only

54
services/indexer/src/crawl.ts
View File

@@ -221,8 +221,17 @@ async function main() {
case 'discover-repos': { case 'discover-repos': {
console.log('Running all discovery strategies...\n'); console.log('Running all discovery strategies...\n');
const discoverDb = createDb(process.env.DATABASE_URL); const discoverDb = createDb(process.env.DATABASE_URL);
const discoverBudgetCrawler = createCrawler();
const orchestrator = createStrategyOrchestrator(); const orchestrator = createStrategyOrchestrator();
// Check budget before starting (reserve 20% for website users)
const discoverBudget = await discoverBudgetCrawler.checkBudget(0.20);
console.log(`API Budget: ${discoverBudget.remaining}/${discoverBudget.limit} remaining (reserve 20%)`);
if (!discoverBudget.ok) {
console.log(`\nAPI budget too low. Waiting for reset...`);
await discoverBudgetCrawler.waitForBudget(0.20);
}
const { repos: discoveredRepos, stats: discoverStats } = await orchestrator.runAllStrategies(); const { repos: discoveredRepos, stats: discoverStats } = await orchestrator.runAllStrategies();
console.log(`\nSaving ${discoveredRepos.length} discovered repos to database...`); console.log(`\nSaving ${discoveredRepos.length} discovered repos to database...`);
@@ -249,8 +258,17 @@ async function main() {
case 'awesome-lists': { case 'awesome-lists': {
console.log('Running awesome list discovery...\n'); console.log('Running awesome list discovery...\n');
const awesomeDb = createDb(process.env.DATABASE_URL); const awesomeDb = createDb(process.env.DATABASE_URL);
const awesomeBudgetCrawler = createCrawler();
const awesomeCrawler = createAwesomeListCrawler(); const awesomeCrawler = createAwesomeListCrawler();
// Check budget before starting (reserve 20% for website users)
const awesomeBudget = await awesomeBudgetCrawler.checkBudget(0.20);
console.log(`API Budget: ${awesomeBudget.remaining}/${awesomeBudget.limit} remaining (reserve 20%)`);
if (!awesomeBudget.ok) {
console.log(`\nAPI budget too low. Waiting for reset...`);
await awesomeBudgetCrawler.waitForBudget(0.20);
}
// Save known lists to DB // Save known lists to DB
for (const list of awesomeCrawler.getKnownLists()) { for (const list of awesomeCrawler.getKnownLists()) {
await awesomeListQueries.upsert(awesomeDb, { await awesomeListQueries.upsert(awesomeDb, {
@@ -297,6 +315,18 @@ async function main() {
const deepCrawler = createDeepScanCrawler(); const deepCrawler = createDeepScanCrawler();
const skillCrawler = createCrawler(); const skillCrawler = createCrawler();
// Parse budget option (default 20% reserve for website users)
const deepBudgetArg = process.argv.find(a => a.startsWith('--budget='));
const deepBudgetPct = deepBudgetArg ? parseInt(deepBudgetArg.split('=')[1]) / 100 : 0.20;
// Check initial budget
const deepInitialBudget = await skillCrawler.checkBudget(deepBudgetPct);
console.log(`API Budget: ${deepInitialBudget.remaining}/${deepInitialBudget.limit} remaining (reserve ${Math.round(deepBudgetPct * 100)}%)`);
if (!deepInitialBudget.ok) {
console.log(`\nAPI budget too low. Waiting for reset...`);
await skillCrawler.waitForBudget(deepBudgetPct);
}
// Get repos that need scanning (never scanned or stale) // Get repos that need scanning (never scanned or stale)
const oneWeekAgo = new Date(); const oneWeekAgo = new Date();
oneWeekAgo.setDate(oneWeekAgo.getDate() - 7); oneWeekAgo.setDate(oneWeekAgo.getDate() - 7);
@@ -307,13 +337,22 @@ async function main() {
break; break;
} }
console.log(`Found ${reposToScan.length} repositories to scan`); console.log(`Found ${reposToScan.length} repositories to scan\n`);
let scannedCount = 0; let scannedCount = 0;
let skillsDiscovered = 0; let skillsDiscovered = 0;
let skillsIndexed = 0; let skillsIndexed = 0;
for (const repo of reposToScan) { for (const repo of reposToScan) {
// Check budget every 10 repos
if (scannedCount > 0 && scannedCount % 10 === 0) {
const midBudget = await skillCrawler.checkBudget(deepBudgetPct);
if (!midBudget.ok) {
console.log(`\n Budget low (${midBudget.remaining}/${midBudget.limit}). Pausing for reset...`);
await skillCrawler.waitForBudget(deepBudgetPct);
}
}
try { try {
console.log(`\nScanning ${repo.owner}/${repo.repo}...`); console.log(`\nScanning ${repo.owner}/${repo.repo}...`);
const skills = await deepCrawler.scanRepository(repo.owner, repo.repo); const skills = await deepCrawler.scanRepository(repo.owner, repo.repo);
@@ -367,6 +406,10 @@ async function main() {
} }
} }
// Show final API status
const deepFinalBudget = await skillCrawler.checkBudget(deepBudgetPct);
console.log(`\n API remaining: ${deepFinalBudget.remaining}/${deepFinalBudget.limit}`);
console.log(`\nDeep scan complete:`); console.log(`\nDeep scan complete:`);
console.log(` Repositories scanned: ${scannedCount}`); console.log(` Repositories scanned: ${scannedCount}`);
console.log(` Skills discovered: ${skillsDiscovered}`); console.log(` Skills discovered: ${skillsDiscovered}`);
@@ -494,6 +537,15 @@ async function main() {
case 'full-enhanced': { case 'full-enhanced': {
console.log('Running full enhanced crawl (discovery + scan + index)...\n'); console.log('Running full enhanced crawl (discovery + scan + index)...\n');
const enhancedDb = createDb(process.env.DATABASE_URL); const enhancedDb = createDb(process.env.DATABASE_URL);
const enhancedBudgetCrawler = createCrawler();
// Check budget before starting (reserve 20% for website users)
const enhancedBudget = await enhancedBudgetCrawler.checkBudget(0.20);
console.log(`API Budget: ${enhancedBudget.remaining}/${enhancedBudget.limit} remaining (reserve 20%)\n`);
if (!enhancedBudget.ok) {
console.log(`API budget too low. Waiting for reset...`);
await enhancedBudgetCrawler.waitForBudget(0.20);
}
// Step 1: Run all discovery strategies // Step 1: Run all discovery strategies
console.log('Step 1: Running discovery strategies...'); console.log('Step 1: Running discovery strategies...');

27
services/indexer/src/crawler.ts
View File

@@ -109,14 +109,10 @@ export class GitHubCrawler {
this.lastCodeSearchTime = Date.now(); this.lastCodeSearchTime = Date.now();
} }
private async getOctokit(): Promise<Octokit> { private async getOctokit(): Promise<{ octokit: Octokit; token: string }> {
return this.octokitPool.getBestInstance(); return this.octokitPool.getBestInstance();
} }
private getCurrentToken(): string {
return this.tokenManager.getBestToken();
}
/** /**
* Discover skills from all sources: official repos, community repos, and GitHub search * Discover skills from all sources: official repos, community repos, and GitHub search
*/ */
@@ -168,8 +164,7 @@ export class GitHubCrawler {
const branch = repoMeta.defaultBranch; const branch = repoMeta.defaultBranch;
// List contents of skills directory // List contents of skills directory
const octokit = await this.getOctokit(); const { octokit, token } = await this.getOctokit();
const token = this.getCurrentToken();
const response = await octokit.repos.getContent({ const response = await octokit.repos.getContent({
owner, owner,
repo, repo,
@@ -221,8 +216,7 @@ export class GitHubCrawler {
*/ */
private async checkFileExists(owner: string, repo: string, path: string, ref: string): Promise<boolean> { private async checkFileExists(owner: string, repo: string, path: string, ref: string): Promise<boolean> {
try { try {
const octokit = await this.getOctokit(); const { octokit, token } = await this.getOctokit();
const token = this.getCurrentToken();
const response = await octokit.repos.getContent({ const response = await octokit.repos.getContent({
owner, owner,
repo, repo,
@@ -319,8 +313,7 @@ export class GitHubCrawler {
// Enforce code search secondary rate limit delay // Enforce code search secondary rate limit delay
await this.waitForCodeSearchSlot(); await this.waitForCodeSearchSlot();
const octokit = await this.getOctokit(); const { octokit, token } = await this.getOctokit();
const token = this.getCurrentToken();
const response = await octokit.search.code({ const response = await octokit.search.code({
q: query, q: query,
per_page: perPage, per_page: perPage,
@@ -440,8 +433,7 @@ export class GitHubCrawler {
* Get repository metadata * Get repository metadata
*/ */
async getRepoMetadata(owner: string, repo: string): Promise<RepoMetadata> { async getRepoMetadata(owner: string, repo: string): Promise<RepoMetadata> {
const octokit = await this.getOctokit(); const { octokit, token } = await this.getOctokit();
const token = this.getCurrentToken();
const response = await octokit.repos.get({ owner, repo }); const response = await octokit.repos.get({ owner, repo });
this.octokitPool.updateStats(token, response.headers); this.octokitPool.updateStats(token, response.headers);
@@ -463,8 +455,7 @@ export class GitHubCrawler {
async fetchOwnerEmail(username: string): Promise<{ email: string | null; source: string | null }> { async fetchOwnerEmail(username: string): Promise<{ email: string | null; source: string | null }> {
try { try {
// Step 1: GitHub Profile API // Step 1: GitHub Profile API
const octokit = await this.getOctokit(); const { octokit, token } = await this.getOctokit();
const token = this.getCurrentToken();
const userResponse = await octokit.users.getByUsername({ username }); const userResponse = await octokit.users.getByUsername({ username });
this.octokitPool.updateStats(token, userResponse.headers); this.octokitPool.updateStats(token, userResponse.headers);
@@ -524,8 +515,7 @@ export class GitHubCrawler {
ref: string ref: string
): Promise<string> { ): Promise<string> {
try { try {
const octokit = await this.getOctokit(); const { octokit, token } = await this.getOctokit();
const token = this.getCurrentToken();
const response = await octokit.repos.getContent({ const response = await octokit.repos.getContent({
owner, owner,
repo, repo,
@@ -558,8 +548,7 @@ export class GitHubCrawler {
ref: string ref: string
): Promise<FileInfo[]> { ): Promise<FileInfo[]> {
try { try {
const octokit = await this.getOctokit(); const { octokit, token } = await this.getOctokit();
const token = this.getCurrentToken();
const response = await octokit.repos.getContent({ const response = await octokit.repos.getContent({
owner, owner,
repo, repo,

4
services/indexer/src/octokit-pool.ts
View File

@@ -29,9 +29,9 @@ export class OctokitPool {
return instance; return instance;
} }
async getBestInstance(): Promise<Octokit> { async getBestInstance(): Promise<{ octokit: Octokit; token: string }> {
const token = await this.tokenManager.checkAndRotate(); const token = await this.tokenManager.checkAndRotate();
return this.getInstance(token); return { octokit: this.getInstance(token), token };
} }
updateStats(token: string, headers: Record<string, unknown>): void { updateStats(token: string, headers: Record<string, unknown>): void {

6
services/indexer/src/strategies/awesome-list.ts
View File

@@ -32,7 +32,7 @@ export class AwesomeListCrawler {
this.octokitPool = new OctokitPool(this.tokenManager); this.octokitPool = new OctokitPool(this.tokenManager);
} }
private async getOctokit(): Promise<Octokit> { private async getOctokit(): Promise<{ octokit: Octokit; token: string }> {
return this.octokitPool.getBestInstance(); return this.octokitPool.getBestInstance();
} }
@@ -73,7 +73,7 @@ export class AwesomeListCrawler {
readmePath = 'README.md' readmePath = 'README.md'
): Promise<RepoReference[]> { ): Promise<RepoReference[]> {
try { try {
const octokit = await this.getOctokit(); const { octokit } = await this.getOctokit();
const response = await octokit.repos.getContent({ const response = await octokit.repos.getContent({
owner, owner,
repo, repo,
@@ -165,7 +165,7 @@ export class AwesomeListCrawler {
for (const query of searchQueries) { for (const query of searchQueries) {
try { try {
const octokit = await this.getOctokit(); const { octokit } = await this.getOctokit();
const response = await octokit.search.repos({ const response = await octokit.search.repos({
q: query, q: query,
sort: 'stars', sort: 'stars',

31
services/indexer/src/strategies/deep-scan.ts
View File

@@ -17,14 +17,10 @@ export class DeepScanCrawler {
this.octokitPool = new OctokitPool(this.tokenManager); this.octokitPool = new OctokitPool(this.tokenManager);
} }
private async getOctokit(): Promise<Octokit> { private async getOctokit(): Promise<{ octokit: Octokit; token: string }> {
return this.octokitPool.getBestInstance(); return this.octokitPool.getBestInstance();
} }
private getCurrentToken(): string {
return this.tokenManager.getBestToken();
}
/** /**
* Deep scan a repository for all SKILL.md files using Git Trees API * Deep scan a repository for all SKILL.md files using Git Trees API
* This is more thorough than code search as it scans the entire repo * This is more thorough than code search as it scans the entire repo
@@ -34,8 +30,7 @@ export class DeepScanCrawler {
try { try {
// Get repository info for default branch // Get repository info for default branch
const octokit = await this.getOctokit(); const { octokit, token } = await this.getOctokit();
const token = this.getCurrentToken();
const repoInfo = await octokit.repos.get({ owner, repo }); const repoInfo = await octokit.repos.get({ owner, repo });
this.octokitPool.updateStats(token, repoInfo.headers); this.octokitPool.updateStats(token, repoInfo.headers);
@@ -101,10 +96,23 @@ export class DeepScanCrawler {
console.log(` Repository ${owner}/${repo} is too large, using fallback scan`); console.log(` Repository ${owner}/${repo} is too large, using fallback scan`);
return this.fallbackScan(owner, repo); return this.fallbackScan(owner, repo);
} }
if (this.isRateLimitError(error)) {
console.log(` Rate limit hit scanning ${owner}/${repo}, waiting for token rotation...`);
await this.tokenManager.checkAndRotate();
// Retry once after rotation
return this.scanRepository(owner, repo);
}
throw error; throw error;
} }
} }
private isRateLimitError(error: unknown): boolean {
if (!(error instanceof Error)) return false;
const status = (error as { status?: number }).status;
if (status === 403 || status === 429) return true;
return error.message.includes('rate limit') || error.message.includes('secondary rate limit');
}
/** /**
* Fallback scan for large repositories - scan known skill directories * Fallback scan for large repositories - scan known skill directories
*/ */
@@ -114,8 +122,7 @@ export class DeepScanCrawler {
for (const basePath of knownPaths) { for (const basePath of knownPaths) {
try { try {
const octokit = await this.getOctokit(); const { octokit, token } = await this.getOctokit();
const token = this.getCurrentToken();
const response = await octokit.repos.getContent({ const response = await octokit.repos.getContent({
owner, owner,
@@ -143,8 +150,7 @@ export class DeepScanCrawler {
const dirs = response.data.filter((item) => item.type === 'dir'); const dirs = response.data.filter((item) => item.type === 'dir');
for (const dir of dirs) { for (const dir of dirs) {
try { try {
const subOctokit = await this.getOctokit(); const { octokit: subOctokit, token: subToken } = await this.getOctokit();
const subToken = this.getCurrentToken();
const subDir = await subOctokit.repos.getContent({ const subDir = await subOctokit.repos.getContent({
owner, owner,
repo, repo,
@@ -223,8 +229,7 @@ export class DeepScanCrawler {
*/ */
async getFileContent(owner: string, repo: string, path: string): Promise<string | null> { async getFileContent(owner: string, repo: string, path: string): Promise<string | null> {
try { try {
const octokit = await this.getOctokit(); const { octokit, token } = await this.getOctokit();
const token = this.getCurrentToken();
const response = await octokit.repos.getContent({ const response = await octokit.repos.getContent({
owner, owner,
repo, repo,

15
services/indexer/src/strategies/fork-network.ts
View File

@@ -24,14 +24,10 @@ export class ForkNetworkCrawler {
this.octokitPool = new OctokitPool(this.tokenManager); this.octokitPool = new OctokitPool(this.tokenManager);
} }
private async getOctokit(): Promise<Octokit> { private async getOctokit(): Promise<{ octokit: Octokit; token: string }> {
return this.octokitPool.getBestInstance(); return this.octokitPool.getBestInstance();
} }
private getCurrentToken(): string {
return this.tokenManager.getBestToken();
}
/** /**
* Get all forks of a repository * Get all forks of a repository
*/ */
@@ -41,8 +37,7 @@ export class ForkNetworkCrawler {
while (page <= maxPages) { while (page <= maxPages) {
try { try {
const octokit = await this.getOctokit(); const { octokit, token } = await this.getOctokit();
const token = this.getCurrentToken();
const response = await octokit.repos.listForks({ const response = await octokit.repos.listForks({
owner, owner,
@@ -121,8 +116,7 @@ export class ForkNetworkCrawler {
*/ */
async getParentRepo(owner: string, repo: string): Promise<{ owner: string; repo: string } | null> { async getParentRepo(owner: string, repo: string): Promise<{ owner: string; repo: string } | null> {
try { try {
const octokit = await this.getOctokit(); const { octokit, token } = await this.getOctokit();
const token = this.getCurrentToken();
const response = await octokit.repos.get({ owner, repo }); const response = await octokit.repos.get({ owner, repo });
this.octokitPool.updateStats(token, response.headers); this.octokitPool.updateStats(token, response.headers);
@@ -152,8 +146,7 @@ export class ForkNetworkCrawler {
// Add root repo // Add root repo
try { try {
const octokit = await this.getOctokit(); const { octokit, token } = await this.getOctokit();
const token = this.getCurrentToken();
const rootInfo = await octokit.repos.get({ owner: rootOwner, repo: rootRepo }); const rootInfo = await octokit.repos.get({ owner: rootOwner, repo: rootRepo });
this.octokitPool.updateStats(token, rootInfo.headers); this.octokitPool.updateStats(token, rootInfo.headers);

15
services/indexer/src/strategies/topic-search.ts
View File

@@ -96,14 +96,10 @@ export class TopicSearchCrawler {
this.octokitPool = new OctokitPool(this.tokenManager); this.octokitPool = new OctokitPool(this.tokenManager);
} }
private async getOctokit(): Promise<Octokit> { private async getOctokit(): Promise<{ octokit: Octokit; token: string }> {
return this.octokitPool.getBestInstance(); return this.octokitPool.getBestInstance();
} }
private getCurrentToken(): string {
return this.tokenManager.getBestToken();
}
/** /**
* Search by all known skill-related topics * Search by all known skill-related topics
*/ */
@@ -117,8 +113,7 @@ export class TopicSearchCrawler {
try { try {
console.log(` Searching topic: ${topic}`); console.log(` Searching topic: ${topic}`);
const octokit = await this.getOctokit(); const { octokit, token } = await this.getOctokit();
const token = this.getCurrentToken();
const response = await octokit.search.repos({ const response = await octokit.search.repos({
q: `topic:${topic}`, q: `topic:${topic}`,
sort: 'stars', sort: 'stars',
@@ -175,8 +170,7 @@ export class TopicSearchCrawler {
try { try {
console.log(` Query: ${query}`); console.log(` Query: ${query}`);
const octokit = await this.getOctokit(); const { octokit, token } = await this.getOctokit();
const token = this.getCurrentToken();
const response = await octokit.search.repos({ const response = await octokit.search.repos({
q: query, q: query,
sort: 'stars', sort: 'stars',
@@ -232,8 +226,7 @@ export class TopicSearchCrawler {
for (let page = 2; page <= maxPages; page++) { for (let page = 2; page <= maxPages; page++) {
try { try {
const octokit = await this.getOctokit(); const { octokit, token } = await this.getOctokit();
const token = this.getCurrentToken();
const response = await octokit.search.repos({ const response = await octokit.search.repos({
q: query, q: query,

10
services/indexer/src/token-manager.ts
View File

@@ -53,7 +53,7 @@ export class TokenManager {
}); });
} }
private async refreshAllTokens(): Promise<void> { async refreshAllTokens(): Promise<void> {
for (const tokenInfo of this.tokens) { for (const tokenInfo of this.tokens) {
await this.refreshRateLimit(tokenInfo.token); await this.refreshRateLimit(tokenInfo.token);
} }
@@ -135,7 +135,7 @@ export class TokenManager {
if (typeof remaining === 'string') { if (typeof remaining === 'string') {
tokenInfo.remaining = parseInt(remaining, 10); tokenInfo.remaining = parseInt(remaining, 10);
tokenInfo.isExhausted = tokenInfo.remaining < 10; tokenInfo.isExhausted = tokenInfo.remaining < 2;
} }
if (typeof reset === 'string') { if (typeof reset === 'string') {
tokenInfo.reset = parseInt(reset, 10) * 1000; tokenInfo.reset = parseInt(reset, 10) * 1000;
@@ -174,8 +174,8 @@ export class TokenManager {
); );
await new Promise((resolve) => setTimeout(resolve, waitTime)); await new Promise((resolve) => setTimeout(resolve, waitTime));
// Refresh rate limit after waiting // Refresh ALL tokens after waiting (others may have reset too)
await this.refreshRateLimit(current); await this.refreshAllTokens();
} }
return current; return current;
@@ -201,7 +201,7 @@ export class TokenManager {
tokenInfo.remaining = core.remaining; tokenInfo.remaining = core.remaining;
tokenInfo.reset = core.reset * 1000; tokenInfo.reset = core.reset * 1000;
tokenInfo.limit = core.limit; tokenInfo.limit = core.limit;
tokenInfo.isExhausted = core.remaining < 10; tokenInfo.isExhausted = core.remaining < 2;
console.log( console.log(
`[${tokenInfo.name}] Refreshed: ${core.remaining}/${core.limit} (resets at ${new Date(tokenInfo.reset).toLocaleTimeString()})` `[${tokenInfo.name}] Refreshed: ${core.remaining}/${core.limit} (resets at ${new Date(tokenInfo.reset).toLocaleTimeString()})`