atlatl/skillhub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(indexer): prevent token exhaustion and reserve 20% budget for website users

Browse Source 
- Lower exhaustion threshold from <10 to <2 (stop wasting remaining tokens)
- Refresh ALL tokens after wait instead of only one
- Fix token/Octokit mismatch by returning token from getBestInstance()
- Add budget checking (20% reserve) to deep-scan, discover-repos, awesome-lists,
  and full-enhanced commands
- Add rate limit error handling with retry in DeepScanCrawler
- Sync translation and query changes

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
airano
2026-02-14 18:29:39 +03:30
parent e8bf89d311
commit 444a6668d0
11 changed files with 101 additions and 364 deletions
Download Patch File Download Diff File Expand all files Collapse all files

295
packages/db/src/queries.ts
View File

@@ -15,9 +15,6 @@ import {
addRequests,
emailSubscriptions,
} from './schema.js';
// #region private-only
import { outreachEmails } from './schema.js';
// #endregion private-only
import type * as schema from './schema.js';
@@ -2027,295 +2024,3 @@ export const emailSubscriptionQueries = {
},
};
// #region private-only
/**
* Outreach email queries
*/
export const outreachQueries = {
/**
* Upsert an outreach record for a GitHub owner
*/
upsert: async (
db: DB,
data: {
githubUsername: string;
email?: string | null;
emailSource?: string | null;
repoCount?: number;
skillCount?: number;
totalStars?: number;
status?: string;
templateData?: typeof outreachEmails.$inferInsert['templateData'];
}
) => {
const result = await db
.insert(outreachEmails)
.values({
id: crypto.randomUUID(),
githubUsername: data.githubUsername,
email: data.email ?? undefined,
emailSource: data.emailSource ?? undefined,
repoCount: data.repoCount ?? 0,
skillCount: data.skillCount ?? 0,
totalStars: data.totalStars ?? 0,
status: data.status ?? (data.email ? 'pending' : 'no_email'),
templateData: data.templateData ?? undefined,
})
.onConflictDoUpdate({
target: outreachEmails.githubUsername,
set: {
...(data.email !== undefined ? { email: data.email } : {}),
...(data.emailSource !== undefined ? { emailSource: data.emailSource } : {}),
...(data.repoCount !== undefined ? { repoCount: data.repoCount } : {}),
...(data.skillCount !== undefined ? { skillCount: data.skillCount } : {}),
...(data.totalStars !== undefined ? { totalStars: data.totalStars } : {}),
...(data.status !== undefined ? { status: data.status } : {}),
...(data.templateData !== undefined ? { templateData: data.templateData } : {}),
updatedAt: new Date(),
},
})
.returning();
return result[0];
},
/**
* Get pending outreach emails (with email, not yet sent)
* Three-tier priority:
* Tier 0: owners with 1..priorityMaxStars (sweet spot, most likely to engage)
* Tier 1: owners with >priorityMaxStars (big orgs, less likely to engage)
* Tier 2: owners with 0 stars (least valuable, sent last)
* Within each tier, sorted by stars descending
*/
getPending: async (db: DB, limit: number = 70, priorityMaxStars: number = 10000) => {
return db
.select()
.from(outreachEmails)
.where(
and(
eq(outreachEmails.status, 'pending'),
sql`${outreachEmails.email} IS NOT NULL`
)
)
.orderBy(
sql`CASE
WHEN ${outreachEmails.totalStars} BETWEEN 1 AND ${priorityMaxStars} THEN 0
WHEN ${outreachEmails.totalStars} > ${priorityMaxStars} THEN 1
ELSE 2
END`,
desc(outreachEmails.totalStars)
)
.limit(limit);
},
/**
* Mark an outreach email as sent
*/
markSent: async (db: DB, githubUsername: string, resendEmailId?: string) => {
return db
.update(outreachEmails)
.set({
status: 'sent',
sentAt: new Date(),
resendEmailId: resendEmailId ?? undefined,
})
.where(eq(outreachEmails.githubUsername, githubUsername));
},
/**
* Reset falsely sent records back to pending (where resend_email_id is null)
* Used to fix records that were marked sent but never actually sent
*/
resetFalselySent: async (db: DB) => {
const result = await db
.update(outreachEmails)
.set({
status: 'pending',
sentAt: null,
})
.where(
and(
eq(outreachEmails.status, 'sent'),
sql`${outreachEmails.resendEmailId} IS NULL`
)
)
.returning({ githubUsername: outreachEmails.githubUsername });
return result.length;
},
/**
* Reset failed records back to pending (e.g. after rate limiting)
* Optionally filter by error message pattern
*/
resetFailed: async (db: DB, errorPattern?: string) => {
const conditions = [eq(outreachEmails.status, 'failed')];
if (errorPattern) {
conditions.push(like(outreachEmails.errorMessage, `%${errorPattern}%`));
}
const result = await db
.update(outreachEmails)
.set({
status: 'pending',
errorMessage: null,
})
.where(and(...conditions))
.returning({ githubUsername: outreachEmails.githubUsername });
return result.length;
},
/**
* Mark an outreach email as failed
*/
markFailed: async (db: DB, githubUsername: string, error: string) => {
return db
.update(outreachEmails)
.set({
status: 'failed',
errorMessage: error,
})
.where(eq(outreachEmails.githubUsername, githubUsername));
},
/**
* Mark an outreach email as unsubscribed
*/
markUnsubscribed: async (db: DB, githubUsername: string) => {
return db
.update(outreachEmails)
.set({ status: 'unsubscribed' })
.where(eq(outreachEmails.githubUsername, githubUsername));
},
/**
* Get outreach stats by status
*/
getStats: async (db: DB) => {
return db
.select({
status: outreachEmails.status,
count: sql<number>`count(*)::int`,
})
.from(outreachEmails)
.groupBy(outreachEmails.status);
},
/**
* Get star distribution for pending outreach emails (with email)
* Useful for understanding the priority tiers
*/
getStarDistribution: async (db: DB) => {
return db.execute(sql`
SELECT
CASE
WHEN total_stars = 0 THEN '0'
WHEN total_stars BETWEEN 1 AND 100 THEN '1-100'
WHEN total_stars BETWEEN 101 AND 500 THEN '101-500'
WHEN total_stars BETWEEN 501 AND 1000 THEN '501-1K'
WHEN total_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
WHEN total_stars BETWEEN 5001 AND 10000 THEN '5K-10K'
WHEN total_stars BETWEEN 10001 AND 50000 THEN '10K-50K'
ELSE '50K+'
END as star_range,
COUNT(*) as total,
COUNT(*) FILTER (WHERE status = 'pending' AND email IS NOT NULL) as sendable,
COUNT(*) FILTER (WHERE status = 'sent') as sent,
COUNT(*) FILTER (WHERE status = 'no_email') as no_email
FROM outreach_emails
GROUP BY
CASE
WHEN total_stars = 0 THEN '0'
WHEN total_stars BETWEEN 1 AND 100 THEN '1-100'
WHEN total_stars BETWEEN 101 AND 500 THEN '101-500'
WHEN total_stars BETWEEN 501 AND 1000 THEN '501-1K'
WHEN total_stars BETWEEN 1001 AND 5000 THEN '1K-5K'
WHEN total_stars BETWEEN 5001 AND 10000 THEN '5K-10K'
WHEN total_stars BETWEEN 10001 AND 50000 THEN '10K-50K'
ELSE '50K+'
END
ORDER BY MIN(total_stars)
`);
},
/**
* Populate outreach_emails from skills table (aggregated by owner)
* Only inserts new owners that don't already exist in outreach_emails
*/
populateFromSkills: async (_db: DB) => {
const rawClient = getRawClientLocal();
const result = await rawClient`
INSERT INTO outreach_emails (id, github_username, repo_count, skill_count, total_stars, status, template_data, created_at, updated_at)
SELECT
gen_random_uuid()::TEXT,
s.github_owner,
COUNT(DISTINCT s.github_repo)::INTEGER,
COUNT(*)::INTEGER,
COALESCE(MAX(s.github_stars), 0),
'pending',
jsonb_build_object(
'repos', (
SELECT jsonb_agg(repo_data ORDER BY repo_data->>'stars' DESC)
FROM (
SELECT jsonb_build_object(
'name', sub.github_repo,
'skillCount', sub.cnt,
'stars', sub.max_stars,
'topSkills', sub.top_skills
) as repo_data
FROM (
SELECT
s2.github_repo,
COUNT(*)::INTEGER as cnt,
MAX(s2.github_stars) as max_stars,
jsonb_agg(
jsonb_build_object('id', s2.id, 'name', s2.name, 'description', LEFT(s2.description, 100))
ORDER BY s2.github_stars DESC
) FILTER (WHERE s2.rn <= 3) as top_skills
FROM (
SELECT *, ROW_NUMBER() OVER (PARTITION BY github_owner, github_repo ORDER BY github_stars DESC) as rn
FROM skills
WHERE github_owner = s.github_owner AND is_blocked = false
) s2
GROUP BY s2.github_repo
) sub
) repo_sub
)
),
NOW(),
NOW()
FROM skills s
WHERE s.is_blocked = false
GROUP BY s.github_owner
ON CONFLICT (github_username) DO UPDATE SET
repo_count = EXCLUDED.repo_count,
skill_count = EXCLUDED.skill_count,
total_stars = EXCLUDED.total_stars,
template_data = EXCLUDED.template_data,
updated_at = NOW()
WHERE outreach_emails.status NOT IN ('sent', 'unsubscribed')
`;
return result.count;
},
/**
* Get by github username
*/
getByUsername: async (db: DB, githubUsername: string) => {
const result = await db
.select()
.from(outreachEmails)
.where(eq(outreachEmails.githubUsername, githubUsername))
.limit(1);
return result[0] ?? null;
},
/**
* Get total skill count (for dynamic numbers in emails)
*/
getTotalSkillCount: async (db: DB): Promise<number> => {
const result = await db
.select({ count: sql<number>`count(*)::int` })
.from(skills)
.where(eq(skills.isBlocked, false));
return result[0]?.count ?? 0;
},
};
// #endregion private-only