atlatl/skillhub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(indexer): prevent token exhaustion and reserve 20% budget for website users

Browse Source 
- Lower exhaustion threshold from <10 to <2 (stop wasting remaining tokens)
- Refresh ALL tokens after wait instead of only one
- Fix token/Octokit mismatch by returning token from getBestInstance()
- Add budget checking (20% reserve) to deep-scan, discover-repos, awesome-lists,
  and full-enhanced commands
- Add rate limit error handling with retry in DeepScanCrawler
- Sync translation and query changes

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
airano
2026-02-14 18:29:39 +03:30
parent e8bf89d311
commit 444a6668d0
11 changed files with 101 additions and 364 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
services/indexer/src/crawler.ts
View File

@@ -109,14 +109,10 @@ export class GitHubCrawler {
this.lastCodeSearchTime = Date.now();
}
private async getOctokit(): Promise<Octokit> {
private async getOctokit(): Promise<{ octokit: Octokit; token: string }> {
return this.octokitPool.getBestInstance();
}
private getCurrentToken(): string {
return this.tokenManager.getBestToken();
}
/**
* Discover skills from all sources: official repos, community repos, and GitHub search
*/
@@ -168,8 +164,7 @@ export class GitHubCrawler {
const branch = repoMeta.defaultBranch;
// List contents of skills directory
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.getContent({
owner,
repo,
@@ -221,8 +216,7 @@ export class GitHubCrawler {
*/
private async checkFileExists(owner: string, repo: string, path: string, ref: string): Promise<boolean> {
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.getContent({
owner,
repo,
@@ -319,8 +313,7 @@ export class GitHubCrawler {
// Enforce code search secondary rate limit delay
await this.waitForCodeSearchSlot();
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.search.code({
q: query,
per_page: perPage,
@@ -440,8 +433,7 @@ export class GitHubCrawler {
* Get repository metadata
*/
async getRepoMetadata(owner: string, repo: string): Promise<RepoMetadata> {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.get({ owner, repo });
this.octokitPool.updateStats(token, response.headers);
@@ -463,8 +455,7 @@ export class GitHubCrawler {
async fetchOwnerEmail(username: string): Promise<{ email: string | null; source: string | null }> {
try {
// Step 1: GitHub Profile API
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const userResponse = await octokit.users.getByUsername({ username });
this.octokitPool.updateStats(token, userResponse.headers);
@@ -524,8 +515,7 @@ export class GitHubCrawler {
ref: string
): Promise<string> {
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.getContent({
owner,
repo,
@@ -558,8 +548,7 @@ export class GitHubCrawler {
ref: string
): Promise<FileInfo[]> {
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.getContent({
owner,
repo,