atlatl/skillhub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(indexer): prevent token exhaustion and reserve 20% budget for website users

Browse Source 
- Lower exhaustion threshold from <10 to <2 (stop wasting remaining tokens)
- Refresh ALL tokens after wait instead of only one
- Fix token/Octokit mismatch by returning token from getBestInstance()
- Add budget checking (20% reserve) to deep-scan, discover-repos, awesome-lists,
  and full-enhanced commands
- Add rate limit error handling with retry in DeepScanCrawler
- Sync translation and query changes

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
airano
2026-02-14 18:29:39 +03:30
parent e8bf89d311
commit 444a6668d0
11 changed files with 101 additions and 364 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
services/indexer/src/crawl.ts
View File

@@ -221,8 +221,17 @@ async function main() {
case 'discover-repos': {
console.log('Running all discovery strategies...\n');
const discoverDb = createDb(process.env.DATABASE_URL);
const discoverBudgetCrawler = createCrawler();
const orchestrator = createStrategyOrchestrator();
// Check budget before starting (reserve 20% for website users)
const discoverBudget = await discoverBudgetCrawler.checkBudget(0.20);
console.log(`API Budget: ${discoverBudget.remaining}/${discoverBudget.limit} remaining (reserve 20%)`);
if (!discoverBudget.ok) {
console.log(`\nAPI budget too low. Waiting for reset...`);
await discoverBudgetCrawler.waitForBudget(0.20);
}
const { repos: discoveredRepos, stats: discoverStats } = await orchestrator.runAllStrategies();
console.log(`\nSaving ${discoveredRepos.length} discovered repos to database...`);
@@ -249,8 +258,17 @@ async function main() {
case 'awesome-lists': {
console.log('Running awesome list discovery...\n');
const awesomeDb = createDb(process.env.DATABASE_URL);
const awesomeBudgetCrawler = createCrawler();
const awesomeCrawler = createAwesomeListCrawler();
// Check budget before starting (reserve 20% for website users)
const awesomeBudget = await awesomeBudgetCrawler.checkBudget(0.20);
console.log(`API Budget: ${awesomeBudget.remaining}/${awesomeBudget.limit} remaining (reserve 20%)`);
if (!awesomeBudget.ok) {
console.log(`\nAPI budget too low. Waiting for reset...`);
await awesomeBudgetCrawler.waitForBudget(0.20);
}
// Save known lists to DB
for (const list of awesomeCrawler.getKnownLists()) {
await awesomeListQueries.upsert(awesomeDb, {
@@ -297,6 +315,18 @@ async function main() {
const deepCrawler = createDeepScanCrawler();
const skillCrawler = createCrawler();
// Parse budget option (default 20% reserve for website users)
const deepBudgetArg = process.argv.find(a => a.startsWith('--budget='));
const deepBudgetPct = deepBudgetArg ? parseInt(deepBudgetArg.split('=')[1]) / 100 : 0.20;
// Check initial budget
const deepInitialBudget = await skillCrawler.checkBudget(deepBudgetPct);
console.log(`API Budget: ${deepInitialBudget.remaining}/${deepInitialBudget.limit} remaining (reserve ${Math.round(deepBudgetPct * 100)}%)`);
if (!deepInitialBudget.ok) {
console.log(`\nAPI budget too low. Waiting for reset...`);
await skillCrawler.waitForBudget(deepBudgetPct);
}
// Get repos that need scanning (never scanned or stale)
const oneWeekAgo = new Date();
oneWeekAgo.setDate(oneWeekAgo.getDate() - 7);
@@ -307,13 +337,22 @@ async function main() {
break;
}
console.log(`Found ${reposToScan.length} repositories to scan`);
console.log(`Found ${reposToScan.length} repositories to scan\n`);
let scannedCount = 0;
let skillsDiscovered = 0;
let skillsIndexed = 0;
for (const repo of reposToScan) {
// Check budget every 10 repos
if (scannedCount > 0 && scannedCount % 10 === 0) {
const midBudget = await skillCrawler.checkBudget(deepBudgetPct);
if (!midBudget.ok) {
console.log(`\n Budget low (${midBudget.remaining}/${midBudget.limit}). Pausing for reset...`);
await skillCrawler.waitForBudget(deepBudgetPct);
}
}
try {
console.log(`\nScanning ${repo.owner}/${repo.repo}...`);
const skills = await deepCrawler.scanRepository(repo.owner, repo.repo);
@@ -367,6 +406,10 @@ async function main() {
}
}
// Show final API status
const deepFinalBudget = await skillCrawler.checkBudget(deepBudgetPct);
console.log(`\n API remaining: ${deepFinalBudget.remaining}/${deepFinalBudget.limit}`);
console.log(`\nDeep scan complete:`);
console.log(` Repositories scanned: ${scannedCount}`);
console.log(` Skills discovered: ${skillsDiscovered}`);
@@ -494,6 +537,15 @@ async function main() {
case 'full-enhanced': {
console.log('Running full enhanced crawl (discovery + scan + index)...\n');
const enhancedDb = createDb(process.env.DATABASE_URL);
const enhancedBudgetCrawler = createCrawler();
// Check budget before starting (reserve 20% for website users)
const enhancedBudget = await enhancedBudgetCrawler.checkBudget(0.20);
console.log(`API Budget: ${enhancedBudget.remaining}/${enhancedBudget.limit} remaining (reserve 20%)\n`);
if (!enhancedBudget.ok) {
console.log(`API budget too low. Waiting for reset...`);
await enhancedBudgetCrawler.waitForBudget(0.20);
}
// Step 1: Run all discovery strategies
console.log('Step 1: Running discovery strategies...');

27
services/indexer/src/crawler.ts
View File

@@ -109,14 +109,10 @@ export class GitHubCrawler {
this.lastCodeSearchTime = Date.now();
}
private async getOctokit(): Promise<Octokit> {
private async getOctokit(): Promise<{ octokit: Octokit; token: string }> {
return this.octokitPool.getBestInstance();
}
private getCurrentToken(): string {
return this.tokenManager.getBestToken();
}
/**
* Discover skills from all sources: official repos, community repos, and GitHub search
*/
@@ -168,8 +164,7 @@ export class GitHubCrawler {
const branch = repoMeta.defaultBranch;
// List contents of skills directory
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.getContent({
owner,
repo,
@@ -221,8 +216,7 @@ export class GitHubCrawler {
*/
private async checkFileExists(owner: string, repo: string, path: string, ref: string): Promise<boolean> {
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.getContent({
owner,
repo,
@@ -319,8 +313,7 @@ export class GitHubCrawler {
// Enforce code search secondary rate limit delay
await this.waitForCodeSearchSlot();
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.search.code({
q: query,
per_page: perPage,
@@ -440,8 +433,7 @@ export class GitHubCrawler {
* Get repository metadata
*/
async getRepoMetadata(owner: string, repo: string): Promise<RepoMetadata> {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.get({ owner, repo });
this.octokitPool.updateStats(token, response.headers);
@@ -463,8 +455,7 @@ export class GitHubCrawler {
async fetchOwnerEmail(username: string): Promise<{ email: string | null; source: string | null }> {
try {
// Step 1: GitHub Profile API
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const userResponse = await octokit.users.getByUsername({ username });
this.octokitPool.updateStats(token, userResponse.headers);
@@ -524,8 +515,7 @@ export class GitHubCrawler {
ref: string
): Promise<string> {
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.getContent({
owner,
repo,
@@ -558,8 +548,7 @@ export class GitHubCrawler {
ref: string
): Promise<FileInfo[]> {
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.getContent({
owner,
repo,

4
services/indexer/src/octokit-pool.ts
View File

@@ -29,9 +29,9 @@ export class OctokitPool {
return instance;
}
async getBestInstance(): Promise<Octokit> {
async getBestInstance(): Promise<{ octokit: Octokit; token: string }> {
const token = await this.tokenManager.checkAndRotate();
return this.getInstance(token);
return { octokit: this.getInstance(token), token };
}
updateStats(token: string, headers: Record<string, unknown>): void {

6
services/indexer/src/strategies/awesome-list.ts
View File

@@ -32,7 +32,7 @@ export class AwesomeListCrawler {
this.octokitPool = new OctokitPool(this.tokenManager);
}
private async getOctokit(): Promise<Octokit> {
private async getOctokit(): Promise<{ octokit: Octokit; token: string }> {
return this.octokitPool.getBestInstance();
}
@@ -73,7 +73,7 @@ export class AwesomeListCrawler {
readmePath = 'README.md'
): Promise<RepoReference[]> {
try {
const octokit = await this.getOctokit();
const { octokit } = await this.getOctokit();
const response = await octokit.repos.getContent({
owner,
repo,
@@ -165,7 +165,7 @@ export class AwesomeListCrawler {
for (const query of searchQueries) {
try {
const octokit = await this.getOctokit();
const { octokit } = await this.getOctokit();
const response = await octokit.search.repos({
q: query,
sort: 'stars',

31
services/indexer/src/strategies/deep-scan.ts
View File

@@ -17,14 +17,10 @@ export class DeepScanCrawler {
this.octokitPool = new OctokitPool(this.tokenManager);
}
private async getOctokit(): Promise<Octokit> {
private async getOctokit(): Promise<{ octokit: Octokit; token: string }> {
return this.octokitPool.getBestInstance();
}
private getCurrentToken(): string {
return this.tokenManager.getBestToken();
}
/**
* Deep scan a repository for all SKILL.md files using Git Trees API
* This is more thorough than code search as it scans the entire repo
@@ -34,8 +30,7 @@ export class DeepScanCrawler {
try {
// Get repository info for default branch
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const repoInfo = await octokit.repos.get({ owner, repo });
this.octokitPool.updateStats(token, repoInfo.headers);
@@ -101,10 +96,23 @@ export class DeepScanCrawler {
console.log(` Repository ${owner}/${repo} is too large, using fallback scan`);
return this.fallbackScan(owner, repo);
}
if (this.isRateLimitError(error)) {
console.log(` Rate limit hit scanning ${owner}/${repo}, waiting for token rotation...`);
await this.tokenManager.checkAndRotate();
// Retry once after rotation
return this.scanRepository(owner, repo);
}
throw error;
}
}
private isRateLimitError(error: unknown): boolean {
if (!(error instanceof Error)) return false;
const status = (error as { status?: number }).status;
if (status === 403 || status === 429) return true;
return error.message.includes('rate limit') || error.message.includes('secondary rate limit');
}
/**
* Fallback scan for large repositories - scan known skill directories
*/
@@ -114,8 +122,7 @@ export class DeepScanCrawler {
for (const basePath of knownPaths) {
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.getContent({
owner,
@@ -143,8 +150,7 @@ export class DeepScanCrawler {
const dirs = response.data.filter((item) => item.type === 'dir');
for (const dir of dirs) {
try {
const subOctokit = await this.getOctokit();
const subToken = this.getCurrentToken();
const { octokit: subOctokit, token: subToken } = await this.getOctokit();
const subDir = await subOctokit.repos.getContent({
owner,
repo,
@@ -223,8 +229,7 @@ export class DeepScanCrawler {
*/
async getFileContent(owner: string, repo: string, path: string): Promise<string | null> {
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.getContent({
owner,
repo,

15
services/indexer/src/strategies/fork-network.ts
View File

@@ -24,14 +24,10 @@ export class ForkNetworkCrawler {
this.octokitPool = new OctokitPool(this.tokenManager);
}
private async getOctokit(): Promise<Octokit> {
private async getOctokit(): Promise<{ octokit: Octokit; token: string }> {
return this.octokitPool.getBestInstance();
}
private getCurrentToken(): string {
return this.tokenManager.getBestToken();
}
/**
* Get all forks of a repository
*/
@@ -41,8 +37,7 @@ export class ForkNetworkCrawler {
while (page <= maxPages) {
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.listForks({
owner,
@@ -121,8 +116,7 @@ export class ForkNetworkCrawler {
*/
async getParentRepo(owner: string, repo: string): Promise<{ owner: string; repo: string } | null> {
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.get({ owner, repo });
this.octokitPool.updateStats(token, response.headers);
@@ -152,8 +146,7 @@ export class ForkNetworkCrawler {
// Add root repo
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const rootInfo = await octokit.repos.get({ owner: rootOwner, repo: rootRepo });
this.octokitPool.updateStats(token, rootInfo.headers);

15
services/indexer/src/strategies/topic-search.ts
View File

@@ -96,14 +96,10 @@ export class TopicSearchCrawler {
this.octokitPool = new OctokitPool(this.tokenManager);
}
private async getOctokit(): Promise<Octokit> {
private async getOctokit(): Promise<{ octokit: Octokit; token: string }> {
return this.octokitPool.getBestInstance();
}
private getCurrentToken(): string {
return this.tokenManager.getBestToken();
}
/**
* Search by all known skill-related topics
*/
@@ -117,8 +113,7 @@ export class TopicSearchCrawler {
try {
console.log(` Searching topic: ${topic}`);
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.search.repos({
q: `topic:${topic}`,
sort: 'stars',
@@ -175,8 +170,7 @@ export class TopicSearchCrawler {
try {
console.log(` Query: ${query}`);
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.search.repos({
q: query,
sort: 'stars',
@@ -232,8 +226,7 @@ export class TopicSearchCrawler {
for (let page = 2; page <= maxPages; page++) {
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.search.repos({
q: query,

10
services/indexer/src/token-manager.ts
View File

@@ -53,7 +53,7 @@ export class TokenManager {
});
}
private async refreshAllTokens(): Promise<void> {
async refreshAllTokens(): Promise<void> {
for (const tokenInfo of this.tokens) {
await this.refreshRateLimit(tokenInfo.token);
}
@@ -135,7 +135,7 @@ export class TokenManager {
if (typeof remaining === 'string') {
tokenInfo.remaining = parseInt(remaining, 10);
tokenInfo.isExhausted = tokenInfo.remaining < 10;
tokenInfo.isExhausted = tokenInfo.remaining < 2;
}
if (typeof reset === 'string') {
tokenInfo.reset = parseInt(reset, 10) * 1000;
@@ -174,8 +174,8 @@ export class TokenManager {
);
await new Promise((resolve) => setTimeout(resolve, waitTime));
// Refresh rate limit after waiting
await this.refreshRateLimit(current);
// Refresh ALL tokens after waiting (others may have reset too)
await this.refreshAllTokens();
}
return current;
@@ -201,7 +201,7 @@ export class TokenManager {
tokenInfo.remaining = core.remaining;
tokenInfo.reset = core.reset * 1000;
tokenInfo.limit = core.limit;
tokenInfo.isExhausted = core.remaining < 10;
tokenInfo.isExhausted = core.remaining < 2;
console.log(
`[${tokenInfo.name}] Refreshed: ${core.remaining}/${core.limit} (resets at ${new Date(tokenInfo.reset).toLocaleTimeString()})`