fix(indexer): prevent token exhaustion and reserve 20% budget for website users
- Lower exhaustion threshold from <10 to <2 (stop wasting remaining tokens) - Refresh ALL tokens after wait instead of only one - Fix token/Octokit mismatch by returning token from getBestInstance() - Add budget checking (20% reserve) to deep-scan, discover-repos, awesome-lists, and full-enhanced commands - Add rate limit error handling with retry in DeepScanCrawler - Sync translation and query changes Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -96,14 +96,10 @@ export class TopicSearchCrawler {
|
||||
this.octokitPool = new OctokitPool(this.tokenManager);
|
||||
}
|
||||
|
||||
private async getOctokit(): Promise<Octokit> {
|
||||
private async getOctokit(): Promise<{ octokit: Octokit; token: string }> {
|
||||
return this.octokitPool.getBestInstance();
|
||||
}
|
||||
|
||||
private getCurrentToken(): string {
|
||||
return this.tokenManager.getBestToken();
|
||||
}
|
||||
|
||||
/**
|
||||
* Search by all known skill-related topics
|
||||
*/
|
||||
@@ -117,8 +113,7 @@ export class TopicSearchCrawler {
|
||||
try {
|
||||
console.log(` Searching topic: ${topic}`);
|
||||
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const { octokit, token } = await this.getOctokit();
|
||||
const response = await octokit.search.repos({
|
||||
q: `topic:${topic}`,
|
||||
sort: 'stars',
|
||||
@@ -175,8 +170,7 @@ export class TopicSearchCrawler {
|
||||
try {
|
||||
console.log(` Query: ${query}`);
|
||||
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const { octokit, token } = await this.getOctokit();
|
||||
const response = await octokit.search.repos({
|
||||
q: query,
|
||||
sort: 'stars',
|
||||
@@ -232,8 +226,7 @@ export class TopicSearchCrawler {
|
||||
|
||||
for (let page = 2; page <= maxPages; page++) {
|
||||
try {
|
||||
const octokit = await this.getOctokit();
|
||||
const token = this.getCurrentToken();
|
||||
const { octokit, token } = await this.getOctokit();
|
||||
|
||||
const response = await octokit.search.repos({
|
||||
q: query,
|
||||
|
||||
Reference in New Issue
Block a user