atlatl/skillhub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(indexer): prevent token exhaustion and reserve 20% budget for website users

Browse Source 
- Lower exhaustion threshold from <10 to <2 (stop wasting remaining tokens)
- Refresh ALL tokens after wait instead of only one
- Fix token/Octokit mismatch by returning token from getBestInstance()
- Add budget checking (20% reserve) to deep-scan, discover-repos, awesome-lists,
  and full-enhanced commands
- Add rate limit error handling with retry in DeepScanCrawler
- Sync translation and query changes

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
airano
2026-02-14 18:29:39 +03:30
parent e8bf89d311
commit 444a6668d0
11 changed files with 101 additions and 364 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
apps/web/messages/en.json
View File

@@ -33,7 +33,7 @@
"docs": "Docs",
"about": "About",
"github": "GitHub",
"sourceCode": "Source Code (Coming Soon)",
"sourceCode": "Source Code",
"sourceCodeShort": "Source Code",
"switchLanguage": "Switch to Farsi"
},
@@ -203,7 +203,7 @@
"attribution": "Attribution",
"manageSkills": "Manage Skills"
},
"sourceCode": "Source Code (Coming Soon)",
"sourceCode": "Source Code",
"copyright": "© {year} SkillHub. Open source under MIT license."
},
"about": {

4
apps/web/messages/fa.json
View File

@@ -33,7 +33,7 @@
"docs": "مستندات",
"about": "درباره ما",
"github": "گیت‌هاب",
"sourceCode": "کد منبع (به زودی)",
"sourceCode": "کد منبع",
"sourceCodeShort": "کد منبع",
"switchLanguage": "تغییر به انگلیسی"
},
@@ -203,7 +203,7 @@
"attribution": "تقدیر و تشکر",
"manageSkills": "مدیریت مهارت‌ها"
},
"sourceCode": "کد منبع (به زودی)",
"sourceCode": "کد منبع",
"copyright": "© {year} SkillHub. کد متن‌باز تحت مجوز MIT."
},
"about": {

295
packages/db/src/queries.ts
View File

@@ -15,9 +15,6 @@ import {
addRequests,
emailSubscriptions,
} from './schema.js';
// #region private-only
import { outreachEmails } from './schema.js';
// #endregion private-only
import type * as schema from './schema.js';
@@ -2027,295 +2024,3 @@ export const emailSubscriptionQueries = {
},
};
// #region private-only
/**
* Outreach email queries
*/
export const outreachQueries = {
/**
* Upsert an outreach record for a GitHub owner
*/
upsert: async (
db: DB,
data: {
githubUsername: string;
email?: string | null;
emailSource?: string | null;
repoCount?: number;
skillCount?: number;
totalStars?: number;
status?: string;
templateData?: typeof outreachEmails.$inferInsert['templateData'];
}
) => {
const result = await db
.insert(outreachEmails)
.values({
id: crypto.randomUUID(),
githubUsername: data.githubUsername,
email: data.email ?? undefined,
emailSource: data.emailSource ?? undefined,
repoCount: data.repoCount ?? 0,
skillCount: data.skillCount ?? 0,
totalStars: data.totalStars ?? 0,
status: data.status ?? (data.email ? 'pending' : 'no_email'),
templateData: data.templateData ?? undefined,
})
.onConflictDoUpdate({
target: outreachEmails.githubUsername,
set: {
...(data.email !== undefined ? { email: data.email } : {}),
...(data.emailSource !== undefined ? { emailSource: data.emailSource } : {}),
...(data.repoCount !== undefined ? { repoCount: data.repoCount } : {}),
...(data.skillCount !== undefined ? { skillCount: data.skillCount } : {}),
...(data.totalStars !== undefined ? { totalStars: data.totalStars } : {}),
...(data.status !== undefined ? { status: data.status } : {}),
...(data.templateData !== undefined ? { templateData: data.templateData } : {}),
updatedAt: new Date(),
},
})
.returning();
return result[0];
},
/**
* Get pending outreach emails (with email, not yet sent)
* Three-tier priority:
* Tier 0: owners with 1..priorityMaxStars (sweet spot, most likely to engage)
* Tier 1: owners with >priorityMaxStars (big orgs, less likely to engage)
* Tier 2: owners with 0 stars (least valuable, sent last)
* Within each tier, sorted by stars descending
*/
getPending: async (db: DB, limit: number = 70, priorityMaxStars: number = 10000) => {
return db
.select()
.from(outreachEmails)
.where(
and(
eq(outreachEmails.status, 'pending'),
sql`${outreachEmails.email} IS NOT NULL`
)
)
.orderBy(
sql`CASE
WHEN ${outreachEmails.totalStars} BETWEEN 1 AND ${priorityMaxStars} THEN 0
WHEN ${outreachEmails.totalStars} > ${priorityMaxStars} THEN 1
ELSE 2
END`,
desc(outreachEmails.totalStars)
)
.limit(limit);
},
/**
* Mark an outreach email as sent
*/
markSent: async (db: DB, githubUsername: string, resendEmailId?: string) => {
return db
.update(outreachEmails)
.set({
status: 'sent',
sentAt: new Date(),
resendEmailId: resendEmailId ?? undefined,
})
.where(eq(outreachEmails.githubUsername, githubUsername));
},
/**
* Reset falsely sent records back to pending (where resend_email_id is null)
* Used to fix records that were marked sent but never actually sent
*/
resetFalselySent: async (db: DB) => {
const result = await db
.update(outreachEmails)
.set({
status: 'pending',
sentAt: null,
})
.where(
and(
eq(outreachEmails.status, 'sent'),
sql`${outreachEmails.resendEmailId} IS NULL`
)
)
.returning({ githubUsername: outreachEmails.githubUsername });
return result.length;
},
/**
* Reset failed records back to pending (e.g. after rate limiting)
* Optionally filter by error message pattern
*/
resetFailed: async (db: DB, errorPattern?: string) => {
const conditions = [eq(outreachEmails.status, 'failed')];
if (errorPattern) {
conditions.push(like(outreachEmails.errorMessage, `%${errorPattern}%`));
}
const result = await db
.update(outreachEmails)
.set({
status: 'pending',
errorMessage: null,
})
.where(and(...conditions))
.returning({ githubUsername: outreachEmails.githubUsername });
return result.length;
},
/**
* Mark an outreach email as failed
*/
markFailed: async (db: DB, githubUsername: string, error: string) => {
return db
.update(outreachEmails)
.set({
status: 'failed',
errorMessage: error,
})
.where(eq(outreachEmails.githubUsername, githubUsername));
},
/**
* Mark an outreach email as unsubscribed
*/
markUnsubscribed: async (db: DB, githubUsername: string) => {
return db
.update(outreachEmails)
.set({ status: 'unsubscribed' })
.where(eq(outreachEmails.githubUsername, githubUsername));
},
/**
* Get outreach stats by status
*/
getStats: async (db: DB) => {
return db
.select({
status: outreachEmails.status,
count: sql<number>`count(*)::int`,
})
.from(outreachEmails)
.groupBy(outreachEmails.status);
},
/**
* Get star distribution for pending outreach emails (with email)
* Useful for understanding the priority tiers
*/
getStarDistribution: async (db: DB) => {
return db.execute(sql`
SELECT
CASE
WHEN total_stars = 0 THEN '0'
WHEN total_stars BETWEEN 1 AND 100 THEN '1-100'
WHEN total_stars BETWEEN 101 AND 500 THEN '101-500'
WHEN total_stars BETWEEN 501 AND 1000 THEN '501-1K'
WHEN total_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
WHEN total_stars BETWEEN 5001 AND 10000 THEN '5K-10K'
WHEN total_stars BETWEEN 10001 AND 50000 THEN '10K-50K'
ELSE '50K+'
END as star_range,
COUNT(*) as total,
COUNT(*) FILTER (WHERE status = 'pending' AND email IS NOT NULL) as sendable,
COUNT(*) FILTER (WHERE status = 'sent') as sent,
COUNT(*) FILTER (WHERE status = 'no_email') as no_email
FROM outreach_emails
GROUP BY
CASE
WHEN total_stars = 0 THEN '0'
WHEN total_stars BETWEEN 1 AND 100 THEN '1-100'
WHEN total_stars BETWEEN 101 AND 500 THEN '101-500'
WHEN total_stars BETWEEN 501 AND 1000 THEN '501-1K'
WHEN total_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
WHEN total_stars BETWEEN 5001 AND 10000 THEN '5K-10K'
WHEN total_stars BETWEEN 10001 AND 50000 THEN '10K-50K'
ELSE '50K+'
END
ORDER BY MIN(total_stars)
`);
},
/**
* Populate outreach_emails from skills table (aggregated by owner)
* Only inserts new owners that don't already exist in outreach_emails
*/
populateFromSkills: async (_db: DB) => {
const rawClient = getRawClientLocal();
const result = await rawClient`
INSERT INTO outreach_emails (id, github_username, repo_count, skill_count, total_stars, status, template_data, created_at, updated_at)
SELECT
gen_random_uuid()::TEXT,
s.github_owner,
COUNT(DISTINCT s.github_repo)::INTEGER,
COUNT(*)::INTEGER,
COALESCE(MAX(s.github_stars), 0),
'pending',
jsonb_build_object(
'repos', (
SELECT jsonb_agg(repo_data ORDER BY repo_data->>'stars' DESC)
FROM (
SELECT jsonb_build_object(
'name', sub.github_repo,
'skillCount', sub.cnt,
'stars', sub.max_stars,
'topSkills', sub.top_skills
) as repo_data
FROM (
SELECT
s2.github_repo,
COUNT(*)::INTEGER as cnt,
MAX(s2.github_stars) as max_stars,
jsonb_agg(
jsonb_build_object('id', s2.id, 'name', s2.name, 'description', LEFT(s2.description, 100))
ORDER BY s2.github_stars DESC
) FILTER (WHERE s2.rn <= 3) as top_skills
FROM (
SELECT *, ROW_NUMBER() OVER (PARTITION BY github_owner, github_repo ORDER BY github_stars DESC) as rn
FROM skills
WHERE github_owner = s.github_owner AND is_blocked = false
) s2
GROUP BY s2.github_repo
) sub
) repo_sub
)
),
NOW(),
NOW()
FROM skills s
WHERE s.is_blocked = false
GROUP BY s.github_owner
ON CONFLICT (github_username) DO UPDATE SET
repo_count = EXCLUDED.repo_count,
skill_count = EXCLUDED.skill_count,
total_stars = EXCLUDED.total_stars,
template_data = EXCLUDED.template_data,
updated_at = NOW()
WHERE outreach_emails.status NOT IN ('sent', 'unsubscribed')
`;
return result.count;
},
/**
* Get by github username
*/
getByUsername: async (db: DB, githubUsername: string) => {
const result = await db
.select()
.from(outreachEmails)
.where(eq(outreachEmails.githubUsername, githubUsername))
.limit(1);
return result[0] ?? null;
},
/**
* Get total skill count (for dynamic numbers in emails)
*/
getTotalSkillCount: async (db: DB): Promise<number> => {
const result = await db
.select({ count: sql<number>`count(*)::int` })
.from(skills)
.where(eq(skills.isBlocked, false));
return result[0]?.count ?? 0;
},
};
// #endregion private-only

54
services/indexer/src/crawl.ts
View File

@@ -221,8 +221,17 @@ async function main() {
case 'discover-repos': {
console.log('Running all discovery strategies...\n');
const discoverDb = createDb(process.env.DATABASE_URL);
const discoverBudgetCrawler = createCrawler();
const orchestrator = createStrategyOrchestrator();
// Check budget before starting (reserve 20% for website users)
const discoverBudget = await discoverBudgetCrawler.checkBudget(0.20);
console.log(`API Budget: ${discoverBudget.remaining}/${discoverBudget.limit} remaining (reserve 20%)`);
if (!discoverBudget.ok) {
console.log(`\nAPI budget too low. Waiting for reset...`);
await discoverBudgetCrawler.waitForBudget(0.20);
}
const { repos: discoveredRepos, stats: discoverStats } = await orchestrator.runAllStrategies();
console.log(`\nSaving ${discoveredRepos.length} discovered repos to database...`);
@@ -249,8 +258,17 @@ async function main() {
case 'awesome-lists': {
console.log('Running awesome list discovery...\n');
const awesomeDb = createDb(process.env.DATABASE_URL);
const awesomeBudgetCrawler = createCrawler();
const awesomeCrawler = createAwesomeListCrawler();
// Check budget before starting (reserve 20% for website users)
const awesomeBudget = await awesomeBudgetCrawler.checkBudget(0.20);
console.log(`API Budget: ${awesomeBudget.remaining}/${awesomeBudget.limit} remaining (reserve 20%)`);
if (!awesomeBudget.ok) {
console.log(`\nAPI budget too low. Waiting for reset...`);
await awesomeBudgetCrawler.waitForBudget(0.20);
}
// Save known lists to DB
for (const list of awesomeCrawler.getKnownLists()) {
await awesomeListQueries.upsert(awesomeDb, {
@@ -297,6 +315,18 @@ async function main() {
const deepCrawler = createDeepScanCrawler();
const skillCrawler = createCrawler();
// Parse budget option (default 20% reserve for website users)
const deepBudgetArg = process.argv.find(a => a.startsWith('--budget='));
const deepBudgetPct = deepBudgetArg ? parseInt(deepBudgetArg.split('=')[1]) / 100 : 0.20;
// Check initial budget
const deepInitialBudget = await skillCrawler.checkBudget(deepBudgetPct);
console.log(`API Budget: ${deepInitialBudget.remaining}/${deepInitialBudget.limit} remaining (reserve ${Math.round(deepBudgetPct * 100)}%)`);
if (!deepInitialBudget.ok) {
console.log(`\nAPI budget too low. Waiting for reset...`);
await skillCrawler.waitForBudget(deepBudgetPct);
}
// Get repos that need scanning (never scanned or stale)
const oneWeekAgo = new Date();
oneWeekAgo.setDate(oneWeekAgo.getDate() - 7);
@@ -307,13 +337,22 @@ async function main() {
break;
}
console.log(`Found ${reposToScan.length} repositories to scan`);
console.log(`Found ${reposToScan.length} repositories to scan\n`);
let scannedCount = 0;
let skillsDiscovered = 0;
let skillsIndexed = 0;
for (const repo of reposToScan) {
// Check budget every 10 repos
if (scannedCount > 0 && scannedCount % 10 === 0) {
const midBudget = await skillCrawler.checkBudget(deepBudgetPct);
if (!midBudget.ok) {
console.log(`\n Budget low (${midBudget.remaining}/${midBudget.limit}). Pausing for reset...`);
await skillCrawler.waitForBudget(deepBudgetPct);
}
}
try {
console.log(`\nScanning ${repo.owner}/${repo.repo}...`);
const skills = await deepCrawler.scanRepository(repo.owner, repo.repo);
@@ -367,6 +406,10 @@ async function main() {
}
}
// Show final API status
const deepFinalBudget = await skillCrawler.checkBudget(deepBudgetPct);
console.log(`\n API remaining: ${deepFinalBudget.remaining}/${deepFinalBudget.limit}`);
console.log(`\nDeep scan complete:`);
console.log(` Repositories scanned: ${scannedCount}`);
console.log(` Skills discovered: ${skillsDiscovered}`);
@@ -494,6 +537,15 @@ async function main() {
case 'full-enhanced': {
console.log('Running full enhanced crawl (discovery + scan + index)...\n');
const enhancedDb = createDb(process.env.DATABASE_URL);
const enhancedBudgetCrawler = createCrawler();
// Check budget before starting (reserve 20% for website users)
const enhancedBudget = await enhancedBudgetCrawler.checkBudget(0.20);
console.log(`API Budget: ${enhancedBudget.remaining}/${enhancedBudget.limit} remaining (reserve 20%)\n`);
if (!enhancedBudget.ok) {
console.log(`API budget too low. Waiting for reset...`);
await enhancedBudgetCrawler.waitForBudget(0.20);
}
// Step 1: Run all discovery strategies
console.log('Step 1: Running discovery strategies...');

27
services/indexer/src/crawler.ts
View File

@@ -109,14 +109,10 @@ export class GitHubCrawler {
this.lastCodeSearchTime = Date.now();
}
private async getOctokit(): Promise<Octokit> {
private async getOctokit(): Promise<{ octokit: Octokit; token: string }> {
return this.octokitPool.getBestInstance();
}
private getCurrentToken(): string {
return this.tokenManager.getBestToken();
}
/**
* Discover skills from all sources: official repos, community repos, and GitHub search
*/
@@ -168,8 +164,7 @@ export class GitHubCrawler {
const branch = repoMeta.defaultBranch;
// List contents of skills directory
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.getContent({
owner,
repo,
@@ -221,8 +216,7 @@ export class GitHubCrawler {
*/
private async checkFileExists(owner: string, repo: string, path: string, ref: string): Promise<boolean> {
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.getContent({
owner,
repo,
@@ -319,8 +313,7 @@ export class GitHubCrawler {
// Enforce code search secondary rate limit delay
await this.waitForCodeSearchSlot();
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.search.code({
q: query,
per_page: perPage,
@@ -440,8 +433,7 @@ export class GitHubCrawler {
* Get repository metadata
*/
async getRepoMetadata(owner: string, repo: string): Promise<RepoMetadata> {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.get({ owner, repo });
this.octokitPool.updateStats(token, response.headers);
@@ -463,8 +455,7 @@ export class GitHubCrawler {
async fetchOwnerEmail(username: string): Promise<{ email: string | null; source: string | null }> {
try {
// Step 1: GitHub Profile API
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const userResponse = await octokit.users.getByUsername({ username });
this.octokitPool.updateStats(token, userResponse.headers);
@@ -524,8 +515,7 @@ export class GitHubCrawler {
ref: string
): Promise<string> {
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.getContent({
owner,
repo,
@@ -558,8 +548,7 @@ export class GitHubCrawler {
ref: string
): Promise<FileInfo[]> {
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.getContent({
owner,
repo,

4
services/indexer/src/octokit-pool.ts
View File

@@ -29,9 +29,9 @@ export class OctokitPool {
return instance;
}
async getBestInstance(): Promise<Octokit> {
async getBestInstance(): Promise<{ octokit: Octokit; token: string }> {
const token = await this.tokenManager.checkAndRotate();
return this.getInstance(token);
return { octokit: this.getInstance(token), token };
}
updateStats(token: string, headers: Record<string, unknown>): void {

6
services/indexer/src/strategies/awesome-list.ts
View File

@@ -32,7 +32,7 @@ export class AwesomeListCrawler {
this.octokitPool = new OctokitPool(this.tokenManager);
}
private async getOctokit(): Promise<Octokit> {
private async getOctokit(): Promise<{ octokit: Octokit; token: string }> {
return this.octokitPool.getBestInstance();
}
@@ -73,7 +73,7 @@ export class AwesomeListCrawler {
readmePath = 'README.md'
): Promise<RepoReference[]> {
try {
const octokit = await this.getOctokit();
const { octokit } = await this.getOctokit();
const response = await octokit.repos.getContent({
owner,
repo,
@@ -165,7 +165,7 @@ export class AwesomeListCrawler {
for (const query of searchQueries) {
try {
const octokit = await this.getOctokit();
const { octokit } = await this.getOctokit();
const response = await octokit.search.repos({
q: query,
sort: 'stars',

31
services/indexer/src/strategies/deep-scan.ts
View File

@@ -17,14 +17,10 @@ export class DeepScanCrawler {
this.octokitPool = new OctokitPool(this.tokenManager);
}
private async getOctokit(): Promise<Octokit> {
private async getOctokit(): Promise<{ octokit: Octokit; token: string }> {
return this.octokitPool.getBestInstance();
}
private getCurrentToken(): string {
return this.tokenManager.getBestToken();
}
/**
* Deep scan a repository for all SKILL.md files using Git Trees API
* This is more thorough than code search as it scans the entire repo
@@ -34,8 +30,7 @@ export class DeepScanCrawler {
try {
// Get repository info for default branch
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const repoInfo = await octokit.repos.get({ owner, repo });
this.octokitPool.updateStats(token, repoInfo.headers);
@@ -101,10 +96,23 @@ export class DeepScanCrawler {
console.log(` Repository ${owner}/${repo} is too large, using fallback scan`);
return this.fallbackScan(owner, repo);
}
if (this.isRateLimitError(error)) {
console.log(` Rate limit hit scanning ${owner}/${repo}, waiting for token rotation...`);
await this.tokenManager.checkAndRotate();
// Retry once after rotation
return this.scanRepository(owner, repo);
}
throw error;
}
}
private isRateLimitError(error: unknown): boolean {
if (!(error instanceof Error)) return false;
const status = (error as { status?: number }).status;
if (status === 403 || status === 429) return true;
return error.message.includes('rate limit') || error.message.includes('secondary rate limit');
}
/**
* Fallback scan for large repositories - scan known skill directories
*/
@@ -114,8 +122,7 @@ export class DeepScanCrawler {
for (const basePath of knownPaths) {
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.getContent({
owner,
@@ -143,8 +150,7 @@ export class DeepScanCrawler {
const dirs = response.data.filter((item) => item.type === 'dir');
for (const dir of dirs) {
try {
const subOctokit = await this.getOctokit();
const subToken = this.getCurrentToken();
const { octokit: subOctokit, token: subToken } = await this.getOctokit();
const subDir = await subOctokit.repos.getContent({
owner,
repo,
@@ -223,8 +229,7 @@ export class DeepScanCrawler {
*/
async getFileContent(owner: string, repo: string, path: string): Promise<string | null> {
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.getContent({
owner,
repo,

15
services/indexer/src/strategies/fork-network.ts
View File

@@ -24,14 +24,10 @@ export class ForkNetworkCrawler {
this.octokitPool = new OctokitPool(this.tokenManager);
}
private async getOctokit(): Promise<Octokit> {
private async getOctokit(): Promise<{ octokit: Octokit; token: string }> {
return this.octokitPool.getBestInstance();
}
private getCurrentToken(): string {
return this.tokenManager.getBestToken();
}
/**
* Get all forks of a repository
*/
@@ -41,8 +37,7 @@ export class ForkNetworkCrawler {
while (page <= maxPages) {
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.listForks({
owner,
@@ -121,8 +116,7 @@ export class ForkNetworkCrawler {
*/
async getParentRepo(owner: string, repo: string): Promise<{ owner: string; repo: string } | null> {
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.repos.get({ owner, repo });
this.octokitPool.updateStats(token, response.headers);
@@ -152,8 +146,7 @@ export class ForkNetworkCrawler {
// Add root repo
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const rootInfo = await octokit.repos.get({ owner: rootOwner, repo: rootRepo });
this.octokitPool.updateStats(token, rootInfo.headers);

15
services/indexer/src/strategies/topic-search.ts
View File

@@ -96,14 +96,10 @@ export class TopicSearchCrawler {
this.octokitPool = new OctokitPool(this.tokenManager);
}
private async getOctokit(): Promise<Octokit> {
private async getOctokit(): Promise<{ octokit: Octokit; token: string }> {
return this.octokitPool.getBestInstance();
}
private getCurrentToken(): string {
return this.tokenManager.getBestToken();
}
/**
* Search by all known skill-related topics
*/
@@ -117,8 +113,7 @@ export class TopicSearchCrawler {
try {
console.log(` Searching topic: ${topic}`);
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.search.repos({
q: `topic:${topic}`,
sort: 'stars',
@@ -175,8 +170,7 @@ export class TopicSearchCrawler {
try {
console.log(` Query: ${query}`);
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.search.repos({
q: query,
sort: 'stars',
@@ -232,8 +226,7 @@ export class TopicSearchCrawler {
for (let page = 2; page <= maxPages; page++) {
try {
const octokit = await this.getOctokit();
const token = this.getCurrentToken();
const { octokit, token } = await this.getOctokit();
const response = await octokit.search.repos({
q: query,

10
services/indexer/src/token-manager.ts
View File

@@ -53,7 +53,7 @@ export class TokenManager {
});
}
private async refreshAllTokens(): Promise<void> {
async refreshAllTokens(): Promise<void> {
for (const tokenInfo of this.tokens) {
await this.refreshRateLimit(tokenInfo.token);
}
@@ -135,7 +135,7 @@ export class TokenManager {
if (typeof remaining === 'string') {
tokenInfo.remaining = parseInt(remaining, 10);
tokenInfo.isExhausted = tokenInfo.remaining < 10;
tokenInfo.isExhausted = tokenInfo.remaining < 2;
}
if (typeof reset === 'string') {
tokenInfo.reset = parseInt(reset, 10) * 1000;
@@ -174,8 +174,8 @@ export class TokenManager {
);
await new Promise((resolve) => setTimeout(resolve, waitTime));
// Refresh rate limit after waiting
await this.refreshRateLimit(current);
// Refresh ALL tokens after waiting (others may have reset too)
await this.refreshAllTokens();
}
return current;
@@ -201,7 +201,7 @@ export class TokenManager {
tokenInfo.remaining = core.remaining;
tokenInfo.reset = core.reset * 1000;
tokenInfo.limit = core.limit;
tokenInfo.isExhausted = core.remaining < 10;
tokenInfo.isExhausted = core.remaining < 2;
console.log(
`[${tokenInfo.name}] Refreshed: ${core.remaining}/${core.limit} (resets at ${new Date(tokenInfo.reset).toLocaleTimeString()})`