diff --git a/apps/web/messages/en.json b/apps/web/messages/en.json index 2cdb4b9..0a6d27d 100644 --- a/apps/web/messages/en.json +++ b/apps/web/messages/en.json @@ -33,7 +33,7 @@ "docs": "Docs", "about": "About", "github": "GitHub", - "sourceCode": "Source Code (Coming Soon)", + "sourceCode": "Source Code", "sourceCodeShort": "Source Code", "switchLanguage": "Switch to Farsi" }, @@ -203,7 +203,7 @@ "attribution": "Attribution", "manageSkills": "Manage Skills" }, - "sourceCode": "Source Code (Coming Soon)", + "sourceCode": "Source Code", "copyright": "© {year} SkillHub. Open source under MIT license." }, "about": { diff --git a/apps/web/messages/fa.json b/apps/web/messages/fa.json index 861933a..f371085 100644 --- a/apps/web/messages/fa.json +++ b/apps/web/messages/fa.json @@ -33,7 +33,7 @@ "docs": "مستندات", "about": "درباره ما", "github": "گیت‌هاب", - "sourceCode": "کد منبع (به زودی)", + "sourceCode": "کد منبع", "sourceCodeShort": "کد منبع", "switchLanguage": "تغییر به انگلیسی" }, @@ -203,7 +203,7 @@ "attribution": "تقدیر و تشکر", "manageSkills": "مدیریت مهارت‌ها" }, - "sourceCode": "کد منبع (به زودی)", + "sourceCode": "کد منبع", "copyright": "© {year} SkillHub. کد متن‌باز تحت مجوز MIT." }, "about": { diff --git a/packages/db/src/queries.ts b/packages/db/src/queries.ts index 702d5f6..b60da1f 100644 --- a/packages/db/src/queries.ts +++ b/packages/db/src/queries.ts @@ -15,9 +15,6 @@ import { addRequests, emailSubscriptions, } from './schema.js'; -// #region private-only -import { outreachEmails } from './schema.js'; -// #endregion private-only import type * as schema from './schema.js'; @@ -2027,295 +2024,3 @@ export const emailSubscriptionQueries = { }, }; -// #region private-only -/** - * Outreach email queries - */ -export const outreachQueries = { - /** - * Upsert an outreach record for a GitHub owner - */ - upsert: async ( - db: DB, - data: { - githubUsername: string; - email?: string | null; - emailSource?: string | null; - repoCount?: number; - skillCount?: number; - totalStars?: number; - status?: string; - templateData?: typeof outreachEmails.$inferInsert['templateData']; - } - ) => { - const result = await db - .insert(outreachEmails) - .values({ - id: crypto.randomUUID(), - githubUsername: data.githubUsername, - email: data.email ?? undefined, - emailSource: data.emailSource ?? undefined, - repoCount: data.repoCount ?? 0, - skillCount: data.skillCount ?? 0, - totalStars: data.totalStars ?? 0, - status: data.status ?? (data.email ? 'pending' : 'no_email'), - templateData: data.templateData ?? undefined, - }) - .onConflictDoUpdate({ - target: outreachEmails.githubUsername, - set: { - ...(data.email !== undefined ? { email: data.email } : {}), - ...(data.emailSource !== undefined ? { emailSource: data.emailSource } : {}), - ...(data.repoCount !== undefined ? { repoCount: data.repoCount } : {}), - ...(data.skillCount !== undefined ? { skillCount: data.skillCount } : {}), - ...(data.totalStars !== undefined ? { totalStars: data.totalStars } : {}), - ...(data.status !== undefined ? { status: data.status } : {}), - ...(data.templateData !== undefined ? { templateData: data.templateData } : {}), - updatedAt: new Date(), - }, - }) - .returning(); - return result[0]; - }, - - /** - * Get pending outreach emails (with email, not yet sent) - * Three-tier priority: - * Tier 0: owners with 1..priorityMaxStars (sweet spot, most likely to engage) - * Tier 1: owners with >priorityMaxStars (big orgs, less likely to engage) - * Tier 2: owners with 0 stars (least valuable, sent last) - * Within each tier, sorted by stars descending - */ - getPending: async (db: DB, limit: number = 70, priorityMaxStars: number = 10000) => { - return db - .select() - .from(outreachEmails) - .where( - and( - eq(outreachEmails.status, 'pending'), - sql`${outreachEmails.email} IS NOT NULL` - ) - ) - .orderBy( - sql`CASE - WHEN ${outreachEmails.totalStars} BETWEEN 1 AND ${priorityMaxStars} THEN 0 - WHEN ${outreachEmails.totalStars} > ${priorityMaxStars} THEN 1 - ELSE 2 - END`, - desc(outreachEmails.totalStars) - ) - .limit(limit); - }, - - /** - * Mark an outreach email as sent - */ - markSent: async (db: DB, githubUsername: string, resendEmailId?: string) => { - return db - .update(outreachEmails) - .set({ - status: 'sent', - sentAt: new Date(), - resendEmailId: resendEmailId ?? undefined, - }) - .where(eq(outreachEmails.githubUsername, githubUsername)); - }, - - /** - * Reset falsely sent records back to pending (where resend_email_id is null) - * Used to fix records that were marked sent but never actually sent - */ - resetFalselySent: async (db: DB) => { - const result = await db - .update(outreachEmails) - .set({ - status: 'pending', - sentAt: null, - }) - .where( - and( - eq(outreachEmails.status, 'sent'), - sql`${outreachEmails.resendEmailId} IS NULL` - ) - ) - .returning({ githubUsername: outreachEmails.githubUsername }); - return result.length; - }, - - /** - * Reset failed records back to pending (e.g. after rate limiting) - * Optionally filter by error message pattern - */ - resetFailed: async (db: DB, errorPattern?: string) => { - const conditions = [eq(outreachEmails.status, 'failed')]; - if (errorPattern) { - conditions.push(like(outreachEmails.errorMessage, `%${errorPattern}%`)); - } - const result = await db - .update(outreachEmails) - .set({ - status: 'pending', - errorMessage: null, - }) - .where(and(...conditions)) - .returning({ githubUsername: outreachEmails.githubUsername }); - return result.length; - }, - - /** - * Mark an outreach email as failed - */ - markFailed: async (db: DB, githubUsername: string, error: string) => { - return db - .update(outreachEmails) - .set({ - status: 'failed', - errorMessage: error, - }) - .where(eq(outreachEmails.githubUsername, githubUsername)); - }, - - /** - * Mark an outreach email as unsubscribed - */ - markUnsubscribed: async (db: DB, githubUsername: string) => { - return db - .update(outreachEmails) - .set({ status: 'unsubscribed' }) - .where(eq(outreachEmails.githubUsername, githubUsername)); - }, - - /** - * Get outreach stats by status - */ - getStats: async (db: DB) => { - return db - .select({ - status: outreachEmails.status, - count: sql`count(*)::int`, - }) - .from(outreachEmails) - .groupBy(outreachEmails.status); - }, - - /** - * Get star distribution for pending outreach emails (with email) - * Useful for understanding the priority tiers - */ - getStarDistribution: async (db: DB) => { - return db.execute(sql` - SELECT - CASE - WHEN total_stars = 0 THEN '0' - WHEN total_stars BETWEEN 1 AND 100 THEN '1-100' - WHEN total_stars BETWEEN 101 AND 500 THEN '101-500' - WHEN total_stars BETWEEN 501 AND 1000 THEN '501-1K' - WHEN total_stars BETWEEN 1001 AND 5000 THEN '1K-5K' - WHEN total_stars BETWEEN 5001 AND 10000 THEN '5K-10K' - WHEN total_stars BETWEEN 10001 AND 50000 THEN '10K-50K' - ELSE '50K+' - END as star_range, - COUNT(*) as total, - COUNT(*) FILTER (WHERE status = 'pending' AND email IS NOT NULL) as sendable, - COUNT(*) FILTER (WHERE status = 'sent') as sent, - COUNT(*) FILTER (WHERE status = 'no_email') as no_email - FROM outreach_emails - GROUP BY - CASE - WHEN total_stars = 0 THEN '0' - WHEN total_stars BETWEEN 1 AND 100 THEN '1-100' - WHEN total_stars BETWEEN 101 AND 500 THEN '101-500' - WHEN total_stars BETWEEN 501 AND 1000 THEN '501-1K' - WHEN total_stars BETWEEN 1001 AND 5000 THEN '1K-5K' - WHEN total_stars BETWEEN 5001 AND 10000 THEN '5K-10K' - WHEN total_stars BETWEEN 10001 AND 50000 THEN '10K-50K' - ELSE '50K+' - END - ORDER BY MIN(total_stars) - `); - }, - - /** - * Populate outreach_emails from skills table (aggregated by owner) - * Only inserts new owners that don't already exist in outreach_emails - */ - populateFromSkills: async (_db: DB) => { - const rawClient = getRawClientLocal(); - const result = await rawClient` - INSERT INTO outreach_emails (id, github_username, repo_count, skill_count, total_stars, status, template_data, created_at, updated_at) - SELECT - gen_random_uuid()::TEXT, - s.github_owner, - COUNT(DISTINCT s.github_repo)::INTEGER, - COUNT(*)::INTEGER, - COALESCE(MAX(s.github_stars), 0), - 'pending', - jsonb_build_object( - 'repos', ( - SELECT jsonb_agg(repo_data ORDER BY repo_data->>'stars' DESC) - FROM ( - SELECT jsonb_build_object( - 'name', sub.github_repo, - 'skillCount', sub.cnt, - 'stars', sub.max_stars, - 'topSkills', sub.top_skills - ) as repo_data - FROM ( - SELECT - s2.github_repo, - COUNT(*)::INTEGER as cnt, - MAX(s2.github_stars) as max_stars, - jsonb_agg( - jsonb_build_object('id', s2.id, 'name', s2.name, 'description', LEFT(s2.description, 100)) - ORDER BY s2.github_stars DESC - ) FILTER (WHERE s2.rn <= 3) as top_skills - FROM ( - SELECT *, ROW_NUMBER() OVER (PARTITION BY github_owner, github_repo ORDER BY github_stars DESC) as rn - FROM skills - WHERE github_owner = s.github_owner AND is_blocked = false - ) s2 - GROUP BY s2.github_repo - ) sub - ) repo_sub - ) - ), - NOW(), - NOW() - FROM skills s - WHERE s.is_blocked = false - GROUP BY s.github_owner - ON CONFLICT (github_username) DO UPDATE SET - repo_count = EXCLUDED.repo_count, - skill_count = EXCLUDED.skill_count, - total_stars = EXCLUDED.total_stars, - template_data = EXCLUDED.template_data, - updated_at = NOW() - WHERE outreach_emails.status NOT IN ('sent', 'unsubscribed') - `; - return result.count; - }, - - /** - * Get by github username - */ - getByUsername: async (db: DB, githubUsername: string) => { - const result = await db - .select() - .from(outreachEmails) - .where(eq(outreachEmails.githubUsername, githubUsername)) - .limit(1); - return result[0] ?? null; - }, - - /** - * Get total skill count (for dynamic numbers in emails) - */ - getTotalSkillCount: async (db: DB): Promise => { - const result = await db - .select({ count: sql`count(*)::int` }) - .from(skills) - .where(eq(skills.isBlocked, false)); - return result[0]?.count ?? 0; - }, -}; -// #endregion private-only diff --git a/services/indexer/src/crawl.ts b/services/indexer/src/crawl.ts index 00fd0bf..31133cf 100644 --- a/services/indexer/src/crawl.ts +++ b/services/indexer/src/crawl.ts @@ -221,8 +221,17 @@ async function main() { case 'discover-repos': { console.log('Running all discovery strategies...\n'); const discoverDb = createDb(process.env.DATABASE_URL); + const discoverBudgetCrawler = createCrawler(); const orchestrator = createStrategyOrchestrator(); + // Check budget before starting (reserve 20% for website users) + const discoverBudget = await discoverBudgetCrawler.checkBudget(0.20); + console.log(`API Budget: ${discoverBudget.remaining}/${discoverBudget.limit} remaining (reserve 20%)`); + if (!discoverBudget.ok) { + console.log(`\nAPI budget too low. Waiting for reset...`); + await discoverBudgetCrawler.waitForBudget(0.20); + } + const { repos: discoveredRepos, stats: discoverStats } = await orchestrator.runAllStrategies(); console.log(`\nSaving ${discoveredRepos.length} discovered repos to database...`); @@ -249,8 +258,17 @@ async function main() { case 'awesome-lists': { console.log('Running awesome list discovery...\n'); const awesomeDb = createDb(process.env.DATABASE_URL); + const awesomeBudgetCrawler = createCrawler(); const awesomeCrawler = createAwesomeListCrawler(); + // Check budget before starting (reserve 20% for website users) + const awesomeBudget = await awesomeBudgetCrawler.checkBudget(0.20); + console.log(`API Budget: ${awesomeBudget.remaining}/${awesomeBudget.limit} remaining (reserve 20%)`); + if (!awesomeBudget.ok) { + console.log(`\nAPI budget too low. Waiting for reset...`); + await awesomeBudgetCrawler.waitForBudget(0.20); + } + // Save known lists to DB for (const list of awesomeCrawler.getKnownLists()) { await awesomeListQueries.upsert(awesomeDb, { @@ -297,6 +315,18 @@ async function main() { const deepCrawler = createDeepScanCrawler(); const skillCrawler = createCrawler(); + // Parse budget option (default 20% reserve for website users) + const deepBudgetArg = process.argv.find(a => a.startsWith('--budget=')); + const deepBudgetPct = deepBudgetArg ? parseInt(deepBudgetArg.split('=')[1]) / 100 : 0.20; + + // Check initial budget + const deepInitialBudget = await skillCrawler.checkBudget(deepBudgetPct); + console.log(`API Budget: ${deepInitialBudget.remaining}/${deepInitialBudget.limit} remaining (reserve ${Math.round(deepBudgetPct * 100)}%)`); + if (!deepInitialBudget.ok) { + console.log(`\nAPI budget too low. Waiting for reset...`); + await skillCrawler.waitForBudget(deepBudgetPct); + } + // Get repos that need scanning (never scanned or stale) const oneWeekAgo = new Date(); oneWeekAgo.setDate(oneWeekAgo.getDate() - 7); @@ -307,13 +337,22 @@ async function main() { break; } - console.log(`Found ${reposToScan.length} repositories to scan`); + console.log(`Found ${reposToScan.length} repositories to scan\n`); let scannedCount = 0; let skillsDiscovered = 0; let skillsIndexed = 0; for (const repo of reposToScan) { + // Check budget every 10 repos + if (scannedCount > 0 && scannedCount % 10 === 0) { + const midBudget = await skillCrawler.checkBudget(deepBudgetPct); + if (!midBudget.ok) { + console.log(`\n Budget low (${midBudget.remaining}/${midBudget.limit}). Pausing for reset...`); + await skillCrawler.waitForBudget(deepBudgetPct); + } + } + try { console.log(`\nScanning ${repo.owner}/${repo.repo}...`); const skills = await deepCrawler.scanRepository(repo.owner, repo.repo); @@ -367,6 +406,10 @@ async function main() { } } + // Show final API status + const deepFinalBudget = await skillCrawler.checkBudget(deepBudgetPct); + console.log(`\n API remaining: ${deepFinalBudget.remaining}/${deepFinalBudget.limit}`); + console.log(`\nDeep scan complete:`); console.log(` Repositories scanned: ${scannedCount}`); console.log(` Skills discovered: ${skillsDiscovered}`); @@ -494,6 +537,15 @@ async function main() { case 'full-enhanced': { console.log('Running full enhanced crawl (discovery + scan + index)...\n'); const enhancedDb = createDb(process.env.DATABASE_URL); + const enhancedBudgetCrawler = createCrawler(); + + // Check budget before starting (reserve 20% for website users) + const enhancedBudget = await enhancedBudgetCrawler.checkBudget(0.20); + console.log(`API Budget: ${enhancedBudget.remaining}/${enhancedBudget.limit} remaining (reserve 20%)\n`); + if (!enhancedBudget.ok) { + console.log(`API budget too low. Waiting for reset...`); + await enhancedBudgetCrawler.waitForBudget(0.20); + } // Step 1: Run all discovery strategies console.log('Step 1: Running discovery strategies...'); diff --git a/services/indexer/src/crawler.ts b/services/indexer/src/crawler.ts index 6f9cf34..a8240ae 100644 --- a/services/indexer/src/crawler.ts +++ b/services/indexer/src/crawler.ts @@ -109,14 +109,10 @@ export class GitHubCrawler { this.lastCodeSearchTime = Date.now(); } - private async getOctokit(): Promise { + private async getOctokit(): Promise<{ octokit: Octokit; token: string }> { return this.octokitPool.getBestInstance(); } - private getCurrentToken(): string { - return this.tokenManager.getBestToken(); - } - /** * Discover skills from all sources: official repos, community repos, and GitHub search */ @@ -168,8 +164,7 @@ export class GitHubCrawler { const branch = repoMeta.defaultBranch; // List contents of skills directory - const octokit = await this.getOctokit(); - const token = this.getCurrentToken(); + const { octokit, token } = await this.getOctokit(); const response = await octokit.repos.getContent({ owner, repo, @@ -221,8 +216,7 @@ export class GitHubCrawler { */ private async checkFileExists(owner: string, repo: string, path: string, ref: string): Promise { try { - const octokit = await this.getOctokit(); - const token = this.getCurrentToken(); + const { octokit, token } = await this.getOctokit(); const response = await octokit.repos.getContent({ owner, repo, @@ -319,8 +313,7 @@ export class GitHubCrawler { // Enforce code search secondary rate limit delay await this.waitForCodeSearchSlot(); - const octokit = await this.getOctokit(); - const token = this.getCurrentToken(); + const { octokit, token } = await this.getOctokit(); const response = await octokit.search.code({ q: query, per_page: perPage, @@ -440,8 +433,7 @@ export class GitHubCrawler { * Get repository metadata */ async getRepoMetadata(owner: string, repo: string): Promise { - const octokit = await this.getOctokit(); - const token = this.getCurrentToken(); + const { octokit, token } = await this.getOctokit(); const response = await octokit.repos.get({ owner, repo }); this.octokitPool.updateStats(token, response.headers); @@ -463,8 +455,7 @@ export class GitHubCrawler { async fetchOwnerEmail(username: string): Promise<{ email: string | null; source: string | null }> { try { // Step 1: GitHub Profile API - const octokit = await this.getOctokit(); - const token = this.getCurrentToken(); + const { octokit, token } = await this.getOctokit(); const userResponse = await octokit.users.getByUsername({ username }); this.octokitPool.updateStats(token, userResponse.headers); @@ -524,8 +515,7 @@ export class GitHubCrawler { ref: string ): Promise { try { - const octokit = await this.getOctokit(); - const token = this.getCurrentToken(); + const { octokit, token } = await this.getOctokit(); const response = await octokit.repos.getContent({ owner, repo, @@ -558,8 +548,7 @@ export class GitHubCrawler { ref: string ): Promise { try { - const octokit = await this.getOctokit(); - const token = this.getCurrentToken(); + const { octokit, token } = await this.getOctokit(); const response = await octokit.repos.getContent({ owner, repo, diff --git a/services/indexer/src/octokit-pool.ts b/services/indexer/src/octokit-pool.ts index a199afc..8416241 100644 --- a/services/indexer/src/octokit-pool.ts +++ b/services/indexer/src/octokit-pool.ts @@ -29,9 +29,9 @@ export class OctokitPool { return instance; } - async getBestInstance(): Promise { + async getBestInstance(): Promise<{ octokit: Octokit; token: string }> { const token = await this.tokenManager.checkAndRotate(); - return this.getInstance(token); + return { octokit: this.getInstance(token), token }; } updateStats(token: string, headers: Record): void { diff --git a/services/indexer/src/strategies/awesome-list.ts b/services/indexer/src/strategies/awesome-list.ts index d637892..ddd0bb9 100644 --- a/services/indexer/src/strategies/awesome-list.ts +++ b/services/indexer/src/strategies/awesome-list.ts @@ -32,7 +32,7 @@ export class AwesomeListCrawler { this.octokitPool = new OctokitPool(this.tokenManager); } - private async getOctokit(): Promise { + private async getOctokit(): Promise<{ octokit: Octokit; token: string }> { return this.octokitPool.getBestInstance(); } @@ -73,7 +73,7 @@ export class AwesomeListCrawler { readmePath = 'README.md' ): Promise { try { - const octokit = await this.getOctokit(); + const { octokit } = await this.getOctokit(); const response = await octokit.repos.getContent({ owner, repo, @@ -165,7 +165,7 @@ export class AwesomeListCrawler { for (const query of searchQueries) { try { - const octokit = await this.getOctokit(); + const { octokit } = await this.getOctokit(); const response = await octokit.search.repos({ q: query, sort: 'stars', diff --git a/services/indexer/src/strategies/deep-scan.ts b/services/indexer/src/strategies/deep-scan.ts index a14410f..da2b6c5 100644 --- a/services/indexer/src/strategies/deep-scan.ts +++ b/services/indexer/src/strategies/deep-scan.ts @@ -17,14 +17,10 @@ export class DeepScanCrawler { this.octokitPool = new OctokitPool(this.tokenManager); } - private async getOctokit(): Promise { + private async getOctokit(): Promise<{ octokit: Octokit; token: string }> { return this.octokitPool.getBestInstance(); } - private getCurrentToken(): string { - return this.tokenManager.getBestToken(); - } - /** * Deep scan a repository for all SKILL.md files using Git Trees API * This is more thorough than code search as it scans the entire repo @@ -34,8 +30,7 @@ export class DeepScanCrawler { try { // Get repository info for default branch - const octokit = await this.getOctokit(); - const token = this.getCurrentToken(); + const { octokit, token } = await this.getOctokit(); const repoInfo = await octokit.repos.get({ owner, repo }); this.octokitPool.updateStats(token, repoInfo.headers); @@ -101,10 +96,23 @@ export class DeepScanCrawler { console.log(` Repository ${owner}/${repo} is too large, using fallback scan`); return this.fallbackScan(owner, repo); } + if (this.isRateLimitError(error)) { + console.log(` Rate limit hit scanning ${owner}/${repo}, waiting for token rotation...`); + await this.tokenManager.checkAndRotate(); + // Retry once after rotation + return this.scanRepository(owner, repo); + } throw error; } } + private isRateLimitError(error: unknown): boolean { + if (!(error instanceof Error)) return false; + const status = (error as { status?: number }).status; + if (status === 403 || status === 429) return true; + return error.message.includes('rate limit') || error.message.includes('secondary rate limit'); + } + /** * Fallback scan for large repositories - scan known skill directories */ @@ -114,8 +122,7 @@ export class DeepScanCrawler { for (const basePath of knownPaths) { try { - const octokit = await this.getOctokit(); - const token = this.getCurrentToken(); + const { octokit, token } = await this.getOctokit(); const response = await octokit.repos.getContent({ owner, @@ -143,8 +150,7 @@ export class DeepScanCrawler { const dirs = response.data.filter((item) => item.type === 'dir'); for (const dir of dirs) { try { - const subOctokit = await this.getOctokit(); - const subToken = this.getCurrentToken(); + const { octokit: subOctokit, token: subToken } = await this.getOctokit(); const subDir = await subOctokit.repos.getContent({ owner, repo, @@ -223,8 +229,7 @@ export class DeepScanCrawler { */ async getFileContent(owner: string, repo: string, path: string): Promise { try { - const octokit = await this.getOctokit(); - const token = this.getCurrentToken(); + const { octokit, token } = await this.getOctokit(); const response = await octokit.repos.getContent({ owner, repo, diff --git a/services/indexer/src/strategies/fork-network.ts b/services/indexer/src/strategies/fork-network.ts index 96ce79f..a1b7ae4 100644 --- a/services/indexer/src/strategies/fork-network.ts +++ b/services/indexer/src/strategies/fork-network.ts @@ -24,14 +24,10 @@ export class ForkNetworkCrawler { this.octokitPool = new OctokitPool(this.tokenManager); } - private async getOctokit(): Promise { + private async getOctokit(): Promise<{ octokit: Octokit; token: string }> { return this.octokitPool.getBestInstance(); } - private getCurrentToken(): string { - return this.tokenManager.getBestToken(); - } - /** * Get all forks of a repository */ @@ -41,8 +37,7 @@ export class ForkNetworkCrawler { while (page <= maxPages) { try { - const octokit = await this.getOctokit(); - const token = this.getCurrentToken(); + const { octokit, token } = await this.getOctokit(); const response = await octokit.repos.listForks({ owner, @@ -121,8 +116,7 @@ export class ForkNetworkCrawler { */ async getParentRepo(owner: string, repo: string): Promise<{ owner: string; repo: string } | null> { try { - const octokit = await this.getOctokit(); - const token = this.getCurrentToken(); + const { octokit, token } = await this.getOctokit(); const response = await octokit.repos.get({ owner, repo }); this.octokitPool.updateStats(token, response.headers); @@ -152,8 +146,7 @@ export class ForkNetworkCrawler { // Add root repo try { - const octokit = await this.getOctokit(); - const token = this.getCurrentToken(); + const { octokit, token } = await this.getOctokit(); const rootInfo = await octokit.repos.get({ owner: rootOwner, repo: rootRepo }); this.octokitPool.updateStats(token, rootInfo.headers); diff --git a/services/indexer/src/strategies/topic-search.ts b/services/indexer/src/strategies/topic-search.ts index 6a93c38..e8304f2 100644 --- a/services/indexer/src/strategies/topic-search.ts +++ b/services/indexer/src/strategies/topic-search.ts @@ -96,14 +96,10 @@ export class TopicSearchCrawler { this.octokitPool = new OctokitPool(this.tokenManager); } - private async getOctokit(): Promise { + private async getOctokit(): Promise<{ octokit: Octokit; token: string }> { return this.octokitPool.getBestInstance(); } - private getCurrentToken(): string { - return this.tokenManager.getBestToken(); - } - /** * Search by all known skill-related topics */ @@ -117,8 +113,7 @@ export class TopicSearchCrawler { try { console.log(` Searching topic: ${topic}`); - const octokit = await this.getOctokit(); - const token = this.getCurrentToken(); + const { octokit, token } = await this.getOctokit(); const response = await octokit.search.repos({ q: `topic:${topic}`, sort: 'stars', @@ -175,8 +170,7 @@ export class TopicSearchCrawler { try { console.log(` Query: ${query}`); - const octokit = await this.getOctokit(); - const token = this.getCurrentToken(); + const { octokit, token } = await this.getOctokit(); const response = await octokit.search.repos({ q: query, sort: 'stars', @@ -232,8 +226,7 @@ export class TopicSearchCrawler { for (let page = 2; page <= maxPages; page++) { try { - const octokit = await this.getOctokit(); - const token = this.getCurrentToken(); + const { octokit, token } = await this.getOctokit(); const response = await octokit.search.repos({ q: query, diff --git a/services/indexer/src/token-manager.ts b/services/indexer/src/token-manager.ts index 9cd64e0..88fffc8 100644 --- a/services/indexer/src/token-manager.ts +++ b/services/indexer/src/token-manager.ts @@ -53,7 +53,7 @@ export class TokenManager { }); } - private async refreshAllTokens(): Promise { + async refreshAllTokens(): Promise { for (const tokenInfo of this.tokens) { await this.refreshRateLimit(tokenInfo.token); } @@ -135,7 +135,7 @@ export class TokenManager { if (typeof remaining === 'string') { tokenInfo.remaining = parseInt(remaining, 10); - tokenInfo.isExhausted = tokenInfo.remaining < 10; + tokenInfo.isExhausted = tokenInfo.remaining < 2; } if (typeof reset === 'string') { tokenInfo.reset = parseInt(reset, 10) * 1000; @@ -174,8 +174,8 @@ export class TokenManager { ); await new Promise((resolve) => setTimeout(resolve, waitTime)); - // Refresh rate limit after waiting - await this.refreshRateLimit(current); + // Refresh ALL tokens after waiting (others may have reset too) + await this.refreshAllTokens(); } return current; @@ -201,7 +201,7 @@ export class TokenManager { tokenInfo.remaining = core.remaining; tokenInfo.reset = core.reset * 1000; tokenInfo.limit = core.limit; - tokenInfo.isExhausted = core.remaining < 10; + tokenInfo.isExhausted = core.remaining < 2; console.log( `[${tokenInfo.name}] Refreshed: ${core.remaining}/${core.limit} (resets at ${new Date(tokenInfo.reset).toLocaleTimeString()})`