fix(indexer): prevent token exhaustion and reserve 20% budget for website users

- Lower exhaustion threshold from <10 to <2 (stop wasting remaining tokens)
- Refresh ALL tokens after wait instead of only one
- Fix token/Octokit mismatch by returning token from getBestInstance()
- Add budget checking (20% reserve) to deep-scan, discover-repos, awesome-lists,
  and full-enhanced commands
- Add rate limit error handling with retry in DeepScanCrawler
- Sync translation and query changes

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
airano
2026-02-14 18:29:39 +03:30
parent e8bf89d311
commit 444a6668d0
11 changed files with 101 additions and 364 deletions

View File

@@ -24,14 +24,10 @@ export class ForkNetworkCrawler {
this.octokitPool = new OctokitPool(this.tokenManager);
}
private async getOctokit(): Promise<Octokit> {
private async getOctokit(): Promise<{ octokit: Octokit; token: string }> {
return this.octokitPool.getBestInstance();
}
private getCurrentToken(): string {
return this.tokenManager.getBestToken();
}
/**
* Get all forks of a repository
*/
@@ -41,8 +37,7 @@ export class ForkNetworkCrawler {
while (page <= maxPages) {
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.listForks({
owner,
@@ -121,8 +116,7 @@ export class ForkNetworkCrawler {
*/
async getParentRepo(owner: string, repo: string): Promise<{ owner: string; repo: string } | null> {
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.get({ owner, repo });
this.octokitPool.updateStats(token, response.headers);
@@ -152,8 +146,7 @@ export class ForkNetworkCrawler {
// Add root repo
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const rootInfo = await octokit.repos.get({ owner: rootOwner, repo: rootRepo });
this.octokitPool.updateStats(token, rootInfo.headers);