fix(indexer): prevent token exhaustion and reserve 20% budget for website users
- Lower exhaustion threshold from <10 to <2 (stop wasting remaining tokens) - Refresh ALL tokens after wait instead of only one - Fix token/Octokit mismatch by returning token from getBestInstance() - Add budget checking (20% reserve) to deep-scan, discover-repos, awesome-lists, and full-enhanced commands - Add rate limit error handling with retry in DeepScanCrawler - Sync translation and query changes Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -24,14 +24,10 @@ export class ForkNetworkCrawler {
|
||||
this.octokitPool = new OctokitPool(this.tokenManager);
|
||||
}
|
||||
|
||||
private async getOctokit(): Promise<Octokit> {
|
||||
private async getOctokit(): Promise<{ octokit: Octokit; token: string }> {
|
||||
return this.octokitPool.getBestInstance();
|
||||
}
|
||||
|
||||
private getCurrentToken(): string {
|
||||
return this.tokenManager.getBestToken();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all forks of a repository
|
||||
*/
|
||||
@@ -41,8 +37,7 @@ export class ForkNetworkCrawler {
|
||||
|
||||
while (page <= maxPages) {
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const { octokit, token } = await this.getOctokit();
|
||||
|
||||
const response = await octokit.repos.listForks({
|
||||
owner,
|
||||
@@ -121,8 +116,7 @@ export class ForkNetworkCrawler {
|
||||
*/
|
||||
async getParentRepo(owner: string, repo: string): Promise<{ owner: string; repo: string } | null> {
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const { octokit, token } = await this.getOctokit();
|
||||
const response = await octokit.repos.get({ owner, repo });
|
||||
this.octokitPool.updateStats(token, response.headers);
|
||||
|
||||
@@ -152,8 +146,7 @@ export class ForkNetworkCrawler {
|
||||
|
||||
// Add root repo
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const { octokit, token } = await this.getOctokit();
|
||||
const rootInfo = await octokit.repos.get({ owner: rootOwner, repo: rootRepo });
|
||||
this.octokitPool.updateStats(token, rootInfo.headers);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user