fix(indexer): prevent token exhaustion and reserve 20% budget for website users

- Lower exhaustion threshold from <10 to <2 (stop wasting remaining tokens)
- Refresh ALL tokens after wait instead of only one
- Fix token/Octokit mismatch by returning token from getBestInstance()
- Add budget checking (20% reserve) to deep-scan, discover-repos, awesome-lists,
  and full-enhanced commands
- Add rate limit error handling with retry in DeepScanCrawler
- Sync translation and query changes

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
airano
2026-02-14 18:29:39 +03:30
parent e8bf89d311
commit 444a6668d0
11 changed files with 101 additions and 364 deletions

View File

@@ -96,14 +96,10 @@ export class TopicSearchCrawler {
this.octokitPool = new OctokitPool(this.tokenManager);
}
private async getOctokit(): Promise<Octokit> {
private async getOctokit(): Promise<{ octokit: Octokit; token: string }> {
return this.octokitPool.getBestInstance();
}
private getCurrentToken(): string {
return this.tokenManager.getBestToken();
}
/**
* Search by all known skill-related topics
*/
@@ -117,8 +113,7 @@ export class TopicSearchCrawler {
try {
console.log(` Searching topic: ${topic}`);
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.search.repos({
q: `topic:${topic}`,
sort: 'stars',
@@ -175,8 +170,7 @@ export class TopicSearchCrawler {
try {
console.log(` Query: ${query}`);
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.search.repos({
q: query,
sort: 'stars',
@@ -232,8 +226,7 @@ export class TopicSearchCrawler {
for (let page = 2; page <= maxPages; page++) {
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.search.repos({
q: query,