sync: multi-branch scanning, discovery strategies, UTF-8 fix, curation improvements

- Add multi-branch scanning to discover skills on non-default branches
- Add popular repos and commits search discovery strategies
- Fix UTF-8 encoding sanitization in skill parser
- Add repo_created_at for threshold-based duplicate tie-breaking
- Add indexer tests and vitest config

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

services/indexer/package.json

@@ -9,7 +9,8 @@
     "worker": "tsx src/worker.ts",
     "crawl": "tsx src/crawl.ts",
     "lint": "eslint src/",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc --noEmit",
+    "test": "vitest"
   },
   "dependencies": {
     "@octokit/rest": "^20.0.2",
@@ -26,6 +27,7 @@
     "@types/node": "^20.10.0",
     "tsup": "^8.0.1",
     "tsx": "^4.7.0",
-    "typescript": "^5.3.0"
+    "typescript": "^5.3.0",
+    "vitest": "^1.2.0"
   }
 }

services/indexer/src/crawl.test.ts

@@ -0,0 +1,55 @@
+import { describe, it, expect } from 'vitest';
+import { parseGitHubRepoUrl } from './crawl.js';
+
+describe('parseGitHubRepoUrl', () => {
+  it('parses plain repo URL', () => {
+    const result = parseGitHubRepoUrl('https://github.com/nuxt/ui');
+    expect(result).toEqual({ owner: 'nuxt', repo: 'ui', branch: null });
+  });
+
+  it('parses URL with /tree/branch', () => {
+    const result = parseGitHubRepoUrl('https://github.com/nuxt/ui/tree/v4');
+    expect(result).toEqual({ owner: 'nuxt', repo: 'ui', branch: 'v4' });
+  });
+
+  it('parses URL with /tree/branch and subdirectory path', () => {
+    const result = parseGitHubRepoUrl('https://github.com/nuxt/ui/tree/v4/skills/nuxt-ui');
+    expect(result).toEqual({ owner: 'nuxt', repo: 'ui', branch: 'v4' });
+  });
+
+  it('parses URL with trailing slash', () => {
+    const result = parseGitHubRepoUrl('https://github.com/owner/repo/');
+    expect(result).toEqual({ owner: 'owner', repo: 'repo', branch: null });
+  });
+
+  it('parses URL with /tree/ but no branch (just tree segment)', () => {
+    const result = parseGitHubRepoUrl('https://github.com/owner/repo/tree/');
+    // parts[3] would be undefined since there's no branch after /tree/
+    expect(result).toEqual({ owner: 'owner', repo: 'repo', branch: null });
+  });
+
+  it('handles http URLs', () => {
+    const result = parseGitHubRepoUrl('http://github.com/owner/repo');
+    expect(result).toEqual({ owner: 'owner', repo: 'repo', branch: null });
+  });
+
+  it('handles release branches with slashes in name', () => {
+    // Note: only first segment after /tree/ is captured as branch
+    const result = parseGitHubRepoUrl('https://github.com/owner/repo/tree/release');
+    expect(result).toEqual({ owner: 'owner', repo: 'repo', branch: 'release' });
+  });
+
+  it('throws on URL with only owner', () => {
+    expect(() => parseGitHubRepoUrl('https://github.com/onlyone')).toThrow('Invalid GitHub URL');
+  });
+
+  it('throws on URL with no path', () => {
+    expect(() => parseGitHubRepoUrl('https://github.com/')).toThrow('Invalid GitHub URL');
+  });
+
+  it('handles repo URL with blob segment (not tree)', () => {
+    const result = parseGitHubRepoUrl('https://github.com/owner/repo/blob/main/README.md');
+    // /blob/ is not /tree/, so no branch extraction
+    expect(result).toEqual({ owner: 'owner', repo: 'repo', branch: null });
+  });
+});

services/indexer/src/crawl.ts

@@ -8,11 +8,39 @@ import { INSTRUCTION_FILE_PATTERNS, type SourceFormat } from 'skillhub-core';
 import { scheduleFullCrawl, scheduleIncrementalCrawl, getQueueStats, getQueue } from './queue.js';
 import { syncAllSkillsToMeilisearch, checkMeilisearchHealth } from './meilisearch-sync.js';
 import { createDb, skillQueries, categoryQueries, discoveredRepoQueries, awesomeListQueries, addRequestQueries, userQueries, sql } from '@skillhub/db';
-import { createStrategyOrchestrator, createDeepScanCrawler, createAwesomeListCrawler } from './strategies/index.js';
+import { createStrategyOrchestrator, createDeepScanCrawler, createAwesomeListCrawler, createPopularReposCrawler, createCommitsSearchCrawler } from './strategies/index.js';
 import { createCrawler } from './crawler.js';
 import { indexSkill } from './skill-indexer.js';
 import { TokenManager } from './token-manager.js';
 
+/**
+ * Parse a GitHub repository URL into owner, repo, and optional branch.
+ * Handles: github.com/owner/repo, github.com/owner/repo/tree/branch,
+ * github.com/owner/repo/tree/branch/some/path
+ */
+export function parseGitHubRepoUrl(urlStr: string): {
+  owner: string;
+  repo: string;
+  branch: string | null;
+} {
+  const url = new URL(urlStr);
+  const parts = url.pathname.split('/').filter(Boolean);
+
+  const owner = parts[0];
+  const repo = parts[1];
+
+  if (!owner || !repo) {
+    throw new Error(`Invalid GitHub URL: ${urlStr}`);
+  }
+
+  // Check for /tree/{branch} path segment
+  if (parts[2] === 'tree' && parts[3]) {
+    return { owner, repo, branch: parts[3] };
+  }
+
+  return { owner, repo, branch: null };
+}
+
 const command = process.argv[2] || 'stats';
 
 async function main() {
@@ -309,6 +337,91 @@ async function main() {
       break;
     }
 
+    case 'discover-popular': {
+      console.log('Discovering popular GitHub repositories...\n');
+      const popDb = createDb(process.env.DATABASE_URL);
+      const popCrawler = createPopularReposCrawler();
+      const popBudgetCrawler = createCrawler();
+
+      // Parse min-stars option (default 1000)
+      const minStarsArg = process.argv.find(a => a.startsWith('--min-stars='));
+      const minStars = minStarsArg ? parseInt(minStarsArg.split('=')[1]) : 1000;
+
+      // Check budget
+      const popBudget = await popBudgetCrawler.checkBudget(0.20);
+      console.log(`API Budget: ${popBudget.remaining}/${popBudget.limit} remaining`);
+      if (!popBudget.ok) {
+        console.log(`\nAPI budget too low. Waiting for reset...`);
+        await popBudgetCrawler.waitForBudget(0.20);
+      }
+
+      const repos = await popCrawler.discoverPopularRepos(minStars);
+
+      console.log(`\nSaving ${repos.length} popular repos to discovered_repos...`);
+      let popSaved = 0;
+      for (const repo of repos) {
+        try {
+          await discoveredRepoQueries.upsert(popDb, {
+            id: `${repo.owner}/${repo.repo}`,
+            owner: repo.owner,
+            repo: repo.repo,
+            discoveredVia: 'popular-scan',
+            githubStars: repo.stars,
+            githubForks: repo.forks,
+            defaultBranch: repo.defaultBranch,
+            isArchived: repo.isArchived,
+          });
+          popSaved++;
+        } catch {
+          // Skip errors
+        }
+      }
+      console.log(`Saved ${popSaved} repositories`);
+      console.log('Run "deep-scan" next to scan their branches for skills');
+      break;
+    }
+
+    case 'commits-search': {
+      console.log('Searching for repos with recent SKILL.md commits...\n');
+      const csDb = createDb(process.env.DATABASE_URL);
+      const csCrawler = createCommitsSearchCrawler();
+      const csBudgetCrawler = createCrawler();
+
+      // Parse days-back option (default 30)
+      const daysBackArg = process.argv.find(a => a.startsWith('--days='));
+      const daysBack = daysBackArg ? parseInt(daysBackArg.split('=')[1]) : 30;
+
+      // Check budget
+      const csBudget = await csBudgetCrawler.checkBudget(0.20);
+      console.log(`API Budget: ${csBudget.remaining}/${csBudget.limit} remaining`);
+      if (!csBudget.ok) {
+        console.log(`\nAPI budget too low. Waiting for reset...`);
+        await csBudgetCrawler.waitForBudget(0.20);
+      }
+
+      const csRepos = await csCrawler.discoverReposFromCommits(daysBack);
+
+      console.log(`\nSaving ${csRepos.length} repos to discovered_repos...`);
+      let csSaved = 0;
+      for (const repo of csRepos) {
+        try {
+          await discoveredRepoQueries.upsert(csDb, {
+            id: `${repo.owner}/${repo.repo}`,
+            owner: repo.owner,
+            repo: repo.repo,
+            discoveredVia: 'commits-search',
+            githubStars: repo.stars,
+          });
+          csSaved++;
+        } catch {
+          // Skip errors
+        }
+      }
+      console.log(`Saved ${csSaved} repositories`);
+      console.log('Run "deep-scan" next to scan their branches for skills');
+      break;
+    }
+
     case 'deep-scan': {
       console.log('Deep scanning discovered repositories...\n');
       const deepDb = createDb(process.env.DATABASE_URL);
@@ -319,6 +432,22 @@ async function main() {
       const deepBudgetArg = process.argv.find(a => a.startsWith('--budget='));
       const deepBudgetPct = deepBudgetArg ? parseInt(deepBudgetArg.split('=')[1]) / 100 : 0.20;
 
+      // Parse branch scanning options
+      const allBranchesFlag = process.argv.includes('--all-branches');
+      const branchesArg = process.argv.find(a => a.startsWith('--branches='));
+      const extraBranchPatterns: string[] = branchesArg
+        ? branchesArg.split('=')[1].split(',').map(p => p.trim()).filter(Boolean)
+        : [];
+      const deepScanOptions = { allBranches: allBranchesFlag, extraBranchPatterns };
+
+      if (allBranchesFlag) {
+        console.log('Branch mode: all branches');
+      } else if (extraBranchPatterns.length > 0) {
+        console.log(`Branch mode: smart + extra patterns [${extraBranchPatterns.join(', ')}]`);
+      } else {
+        console.log('Branch mode: smart (default + version/release branches)');
+      }
+
       // Check initial budget
       const deepInitialBudget = await skillCrawler.checkBudget(deepBudgetPct);
       console.log(`API Budget: ${deepInitialBudget.remaining}/${deepInitialBudget.limit} remaining (reserve ${Math.round(deepBudgetPct * 100)}%)`);
@@ -355,7 +484,7 @@ async function main() {
 
         try {
           console.log(`\nScanning ${repo.owner}/${repo.repo}...`);
-          const skills = await deepCrawler.scanRepository(repo.owner, repo.repo);
+          const skills = await deepCrawler.scanRepository(repo.owner, repo.repo, deepScanOptions);
 
           // Mark as scanned
           await discoveredRepoQueries.markScanned(
@@ -646,19 +775,15 @@ async function main() {
         }
 
         try {
-          // Parse repository URL
-          const url = new URL(request.repositoryUrl);
-          const pathParts = url.pathname.split('/').filter(Boolean);
-          const owner = pathParts[0];
-          const repo = pathParts[1];
+          // Parse repository URL (may include branch in /tree/{branch} path)
+          const { owner, repo, branch: urlBranch } = parseGitHubRepoUrl(request.repositoryUrl);
 
-          if (!owner || !repo) {
-            throw new Error('Invalid repository URL');
-          }
-
-          // Get default branch
+          // Get repo metadata for default branch
           const repoMeta = await addCrawler.getRepoMetadata(owner, repo);
-          const branch = repoMeta.defaultBranch;
+          const branch = urlBranch ?? repoMeta.defaultBranch;
+          if (urlBranch) {
+            console.log(`  Branch: ${branch} (from URL)`);
+          }
 
           // Parse skill paths (comma-separated)
           const skillPaths = request.skillPath.split(',').map((p: string) => p.trim());

services/indexer/src/crawler.ts

@@ -45,6 +45,7 @@ export interface RepoMetadata {
   license: string | null;
   description: string | null;
   updatedAt: string;
+  createdAt: string;
   defaultBranch: string;
   topics: string[];
 }
@@ -345,7 +346,7 @@ export class GitHubCrawler {
             owner: item.repository.owner.login,
             repo: item.repository.name,
             path: skillPath,
-            branch: 'main',
+            branch: '',  // empty → fetchSkillContent falls back to repoMeta.defaultBranch
             sourceFormat: expectedFormat,
           });
         }
@@ -443,6 +444,7 @@ export class GitHubCrawler {
       license: response.data.license?.spdx_id || null,
       description: response.data.description,
       updatedAt: response.data.updated_at,
+      createdAt: response.data.created_at,
       defaultBranch: response.data.default_branch,
       topics: response.data.topics || [],
     };

services/indexer/src/skill-indexer.ts

@@ -89,6 +89,7 @@ export async function indexSkill(
     triggers: analysis.skill.metadata.triggers,
     githubStars: content.repoMeta.stars,
     githubForks: content.repoMeta.forks,
+    repoCreatedAt: new Date(content.repoMeta.createdAt),
     securityScore: analysis.security.score,
     securityStatus: analysis.security.status,
     qualityScore: analysis.quality.overall,

services/indexer/src/strategies/commits-search.ts

@@ -0,0 +1,146 @@
+import type { Octokit } from '@octokit/rest';
+import { TokenManager } from '../token-manager.js';
+import { OctokitPool } from '../octokit-pool.js';
+
+export interface CommitSearchRepoResult {
+  owner: string;
+  repo: string;
+  stars?: number;
+}
+
+/**
+ * Commits Search Discovery Strategy
+ * Searches GitHub for recent commits mentioning "SKILL.md" in their messages.
+ * This finds repos where someone recently added or modified a SKILL.md file,
+ * even on non-default branches (unlike Code Search which only indexes default branches).
+ *
+ * Limitation: Searches commit messages, not file paths. Only catches commits
+ * where the author mentioned "SKILL.md" in the message.
+ */
+export class CommitsSearchCrawler {
+  private octokitPool: OctokitPool;
+  private tokenManager: TokenManager;
+
+  constructor(tokenManager?: TokenManager) {
+    this.tokenManager = tokenManager || TokenManager.getInstance();
+    this.octokitPool = new OctokitPool(this.tokenManager);
+  }
+
+  private async getOctokit(): Promise<{ octokit: Octokit; token: string }> {
+    return this.octokitPool.getBestInstance();
+  }
+
+  /**
+   * Search for recent commits mentioning SKILL.md in their messages.
+   * Returns unique repos found.
+   */
+  async discoverReposFromCommits(daysBack: number = 30): Promise<CommitSearchRepoResult[]> {
+    const sinceDate = new Date();
+    sinceDate.setDate(sinceDate.getDate() - daysBack);
+    const dateStr = sinceDate.toISOString().split('T')[0]; // YYYY-MM-DD
+
+    const queries = [
+      `SKILL.md committer-date:>${dateStr}`,
+      `"add SKILL.md" committer-date:>${dateStr}`,
+      `"SKILL.md" path:skills committer-date:>${dateStr}`,
+    ];
+
+    const allRepos = new Map<string, CommitSearchRepoResult>();
+
+    console.log(`Searching commits from last ${daysBack} days (since ${dateStr})...`);
+
+    for (const query of queries) {
+      try {
+        console.log(`\n  Query: ${query}`);
+        const repos = await this.searchCommits(query);
+        for (const repo of repos) {
+          const key = `${repo.owner}/${repo.repo}`.toLowerCase();
+          if (!allRepos.has(key)) {
+            allRepos.set(key, repo);
+          }
+        }
+        console.log(`  Found ${repos.length} repos (total unique: ${allRepos.size})`);
+      } catch (error) {
+        console.warn(`  Failed for query "${query}":`, error instanceof Error ? error.message : error);
+      }
+    }
+
+    console.log(`\nTotal unique repos from commits search: ${allRepos.size}`);
+    return Array.from(allRepos.values());
+  }
+
+  /**
+   * Execute a single commits search query and extract unique repos.
+   */
+  private async searchCommits(query: string): Promise<CommitSearchRepoResult[]> {
+    const repos = new Map<string, CommitSearchRepoResult>();
+    const maxPages = 5;
+    const perPage = 100;
+
+    for (let page = 1; page <= maxPages; page++) {
+      try {
+        const { octokit, token } = await this.getOctokit();
+        const response = await octokit.search.commits({
+          q: query,
+          sort: 'committer-date',
+          order: 'desc',
+          per_page: perPage,
+          page,
+        });
+        this.octokitPool.updateStats(token, response.headers);
+
+        if (page === 1) {
+          console.log(`    Total results: ${response.data.total_count}`);
+        }
+
+        for (const item of response.data.items) {
+          const repoData = item.repository;
+          const key = `${repoData.owner.login}/${repoData.name}`.toLowerCase();
+          if (!repos.has(key)) {
+            repos.set(key, {
+              owner: repoData.owner.login,
+              repo: repoData.name,
+            });
+          }
+        }
+
+        if (response.data.items.length < perPage) break;
+      } catch (error) {
+        if (this.isBeyondResultsLimit(error)) {
+          console.log(`    Reached 1000-result limit`);
+          break;
+        }
+        if (this.isRateLimitError(error)) {
+          console.log(`    Rate limit hit, rotating token...`);
+          await this.tokenManager.checkAndRotate();
+          page--; // Retry same page
+          continue;
+        }
+        throw error;
+      }
+    }
+
+    return Array.from(repos.values());
+  }
+
+  private isRateLimitError(error: unknown): boolean {
+    if (!(error instanceof Error)) return false;
+    const status = (error as { status?: number }).status;
+    return status === 403 || status === 429;
+  }
+
+  private isBeyondResultsLimit(error: unknown): boolean {
+    return (
+      error instanceof Error &&
+      'status' in error &&
+      (error as { status: number }).status === 422
+    );
+  }
+}
+
+export function createCommitsSearchCrawler(tokenManager?: TokenManager | string): CommitsSearchCrawler {
+  if (typeof tokenManager === 'string') {
+    return new CommitsSearchCrawler(new TokenManager([tokenManager]));
+  }
+  return new CommitsSearchCrawler(tokenManager);
+}

services/indexer/src/strategies/deep-scan.test.ts

@@ -0,0 +1,147 @@
+import { describe, it, expect } from 'vitest';
+import { filterAndSortBranches } from './deep-scan.js';
+
+describe('filterAndSortBranches', () => {
+  it('returns only defaultBranch when no matching branches exist', () => {
+    const result = filterAndSortBranches(
+      ['main', 'feature/foo', 'bugfix/bar', 'experiment'],
+      'main'
+    );
+    expect(result).toEqual(['main']);
+  });
+
+  it('includes well-known branch names (up to 5 non-default cap)', () => {
+    const result = filterAndSortBranches(
+      ['main', 'stable', 'next', 'latest', 'canary', 'dev', 'develop', 'feature/x'],
+      'main'
+    );
+    expect(result[0]).toBe('main');
+    expect(result).toContain('stable');
+    expect(result).toContain('next');
+    expect(result).toContain('dev');
+    // 6 well-known names but only 5 non-default slots, so some get dropped
+    expect(result.length).toBe(6); // main + 5
+    expect(result).not.toContain('feature/x');
+  });
+
+  it('includes version branches matching /^[vV]\\d/', () => {
+    const result = filterAndSortBranches(
+      ['main', 'v4', 'v3', 'v2', 'v1'],
+      'main'
+    );
+    expect(result[0]).toBe('main');
+    expect(result).toContain('v4');
+    expect(result).toContain('v3');
+    expect(result).toContain('v2');
+    expect(result).toContain('v1');
+  });
+
+  it('sorts version branches by semver descending', () => {
+    const result = filterAndSortBranches(
+      ['main', 'v1', 'v3', 'v2', 'v10', 'v4'],
+      'main'
+    );
+    const versions = result.filter(b => b.startsWith('v'));
+    expect(versions[0]).toBe('v10');
+    expect(versions[1]).toBe('v4');
+    expect(versions[2]).toBe('v3');
+    expect(versions[3]).toBe('v2');
+  });
+
+  it('handles complex version numbers (v2.1, v3.0.1)', () => {
+    const result = filterAndSortBranches(
+      ['main', 'v2.1', 'v2.0', 'v3.0.1', 'v1.5'],
+      'main'
+    );
+    const versions = result.filter(b => b.startsWith('v'));
+    expect(versions[0]).toBe('v3.0.1');
+    expect(versions[1]).toBe('v2.1');
+    expect(versions[2]).toBe('v2.0');
+    expect(versions[3]).toBe('v1.5');
+  });
+
+  it('takes only top 5 version branches', () => {
+    const result = filterAndSortBranches(
+      ['main', 'v1', 'v2', 'v3', 'v4', 'v5', 'v6', 'v7', 'v8'],
+      'main'
+    );
+    // defaultBranch + 5 non-default max
+    expect(result.length).toBeLessThanOrEqual(6);
+    // Should have the top 5 versions (v8, v7, v6, v5, v4)
+    expect(result).toContain('v8');
+    expect(result).toContain('v7');
+    expect(result).toContain('v6');
+    expect(result).toContain('v5');
+    expect(result).toContain('v4');
+    expect(result).not.toContain('v1');
+  });
+
+  it('caps total non-default at 5 when mixing well-known + versions', () => {
+    const result = filterAndSortBranches(
+      ['main', 'stable', 'next', 'dev', 'v4', 'v3', 'v2', 'v1'],
+      'main'
+    );
+    // 1 default + max 5 non-default
+    expect(result.length).toBeLessThanOrEqual(6);
+    expect(result[0]).toBe('main');
+  });
+
+  it('includes release/ and releases/ prefixed branches', () => {
+    const result = filterAndSortBranches(
+      ['main', 'release/3.0', 'releases/v2', 'feature/x'],
+      'main'
+    );
+    expect(result).toContain('release/3.0');
+    expect(result).toContain('releases/v2');
+    expect(result).not.toContain('feature/x');
+  });
+
+  it('applies extra patterns from CLI (exact and prefix match)', () => {
+    const result = filterAndSortBranches(
+      ['main', 'alpha', 'beta', 'alpha/test', 'feature/x'],
+      'main',
+      ['alpha', 'beta']
+    );
+    expect(result).toContain('alpha');
+    expect(result).toContain('beta');
+    expect(result).not.toContain('feature/x');
+  });
+
+  it('excludes defaultBranch from filtered list even if it matches a pattern', () => {
+    const result = filterAndSortBranches(
+      ['dev', 'v4', 'stable'],
+      'dev'
+    );
+    expect(result[0]).toBe('dev');
+    // dev should appear only once
+    const devCount = result.filter(b => b === 'dev').length;
+    expect(devCount).toBe(1);
+  });
+
+  it('puts defaultBranch first always', () => {
+    const result = filterAndSortBranches(
+      ['v10', 'master', 'v5', 'stable'],
+      'master'
+    );
+    expect(result[0]).toBe('master');
+  });
+
+  it('handles V (uppercase) version branches', () => {
+    const result = filterAndSortBranches(
+      ['main', 'V4', 'V3'],
+      'main'
+    );
+    expect(result).toContain('V4');
+    expect(result).toContain('V3');
+  });
+
+  it('returns [defaultBranch] for empty branch list', () => {
+    const result = filterAndSortBranches([], 'main');
+    expect(result).toEqual(['main']);
+  });
+
+  it('returns [defaultBranch] when only default is in the list', () => {
+    const result = filterAndSortBranches(['main'], 'main');
+    expect(result).toEqual(['main']);
+  });
+});

services/indexer/src/strategies/deep-scan.ts

@@ -3,10 +3,92 @@ import { TokenManager } from '../token-manager.js';import { OctokitPool } from '
 import type { SkillSource } from 'skillhub-core';
 import { INSTRUCTION_FILE_PATTERNS, type SourceFormat } from 'skillhub-core';
 
+export interface DeepScanOptions {
+  allBranches?: boolean;
+  extraBranchPatterns?: string[];
+}
+
+/** Well-known branch names that indicate important non-default branches. */
+const IMPORTANT_BRANCH_NAMES = ['stable', 'next', 'latest', 'canary', 'dev', 'develop'];
+
+/** Prefixes that indicate release/version branches. */
+const IMPORTANT_BRANCH_PREFIXES = ['release/', 'releases/'];
+
+/**
+ * Pure helper: filter and sort branches to find important ones worth scanning.
+ *
+ * Rules:
+ *   1. Always include defaultBranch first
+ *   2. Include exact name matches from IMPORTANT_BRANCH_NAMES
+ *   3. Include prefix matches from IMPORTANT_BRANCH_PREFIXES
+ *   4. Include version branches matching /^[vV]\d/ — sorted by semver desc, top 5
+ *   5. Include branches matching extraPatterns (exact or prefix match)
+ *   6. Cap non-default branches at 5
+ */
+export function filterAndSortBranches(
+  allBranchNames: string[],
+  defaultBranch: string,
+  extraPatterns: string[] = []
+): string[] {
+  const versionBranches: string[] = [];
+  const otherImportant: string[] = [];
+
+  for (const name of allBranchNames) {
+    if (name === defaultBranch) continue;
+
+    // Check well-known exact names
+    if (IMPORTANT_BRANCH_NAMES.includes(name)) {
+      otherImportant.push(name);
+      continue;
+    }
+
+    // Check prefix patterns (release/, releases/)
+    const lower = name.toLowerCase();
+    if (IMPORTANT_BRANCH_PREFIXES.some(p => lower.startsWith(p))) {
+      otherImportant.push(name);
+      continue;
+    }
+
+    // Version branches: v* or V* followed by a digit (e.g., v4, v2.1, V3)
+    if (/^[vV]\d/.test(name)) {
+      versionBranches.push(name);
+      continue;
+    }
+
+    // Check extra patterns from CLI (exact or prefix match)
+    if (extraPatterns.length > 0) {
+      const matched = extraPatterns.some(p => name === p || name.startsWith(p + '/'));
+      if (matched) {
+        otherImportant.push(name);
+      }
+    }
+  }
+
+  // Sort version branches by semver descending, take top 5
+  const sortedVersionBranches = versionBranches
+    .sort((a, b) => {
+      const normalize = (s: string) =>
+        s.replace(/^[vV]/, '').split(/[.\-x]/).map(n => parseInt(n) || 0);
+      const aN = normalize(a);
+      const bN = normalize(b);
+      for (let i = 0; i < Math.max(aN.length, bN.length); i++) {
+        const diff = (bN[i] || 0) - (aN[i] || 0);
+        if (diff !== 0) return diff;
+      }
+      return 0;
+    })
+    .slice(0, 5);
+
+  // Combine: other important first, then version branches, cap at 5 non-default total
+  const nonDefault = [...otherImportant, ...sortedVersionBranches].slice(0, 5);
+  return [defaultBranch, ...nonDefault];
+}
+
 /**
  * Deep Scan Strategy
  * Uses Git Trees API to recursively scan entire repositories for SKILL.md files
  * Can discover skills that aren't found by code search
+ * Supports scanning multiple branches per repository
  */
 export class DeepScanCrawler {
   private octokitPool: OctokitPool;
@@ -22,35 +104,98 @@ export class DeepScanCrawler {
   }
 
   /**
-   * Deep scan a repository for all SKILL.md files using Git Trees API
-   * This is more thorough than code search as it scans the entire repo
+   * List important branches worth scanning beyond the default branch.
+   * Calls octokit.repos.listBranches (1 API call).
    */
-  async scanRepository(owner: string, repo: string): Promise<SkillSource[]> {
+  private async listImportantBranches(
+    owner: string,
+    repo: string,
+    defaultBranch: string,
+    extraPatterns: string[] = []
+  ): Promise<string[]> {
+    try {
+      const { octokit, token } = await this.getOctokit();
+      const response = await octokit.repos.listBranches({
+        owner,
+        repo,
+        per_page: 100,
+      });
+      this.octokitPool.updateStats(token, response.headers);
+
+      const allBranchNames = response.data.map(b => b.name);
+      return filterAndSortBranches(allBranchNames, defaultBranch, extraPatterns);
+    } catch {
+      console.log(`  Could not list branches for ${owner}/${repo}, using default only`);
+      return [defaultBranch];
+    }
+  }
+
+  /**
+   * List all branches in a repository (for --all-branches mode).
+   * Handles pagination for repos with many branches.
+   */
+  private async listAllBranches(
+    owner: string,
+    repo: string,
+    defaultBranch: string
+  ): Promise<string[]> {
+    const allBranches: string[] = [defaultBranch];
+    let page = 1;
+
+    for (;;) {
+      try {
+        const { octokit, token } = await this.getOctokit();
+        const response = await octokit.repos.listBranches({
+          owner,
+          repo,
+          per_page: 100,
+          page,
+        });
+        this.octokitPool.updateStats(token, response.headers);
+
+        for (const branch of response.data) {
+          if (branch.name !== defaultBranch) {
+            allBranches.push(branch.name);
+          }
+        }
+
+        if (response.data.length < 100) break;
+        page++;
+      } catch {
+        break;
+      }
+    }
+
+    console.log(`  Found ${allBranches.length} total branches for ${owner}/${repo}`);
+    return allBranches;
+  }
+
+  /**
+   * Scan a single branch for instruction files using the Git Trees API.
+   * Returns [] silently for truncated trees (fallback handles those at repo level).
+   */
+  private async scanSingleBranch(
+    owner: string,
+    repo: string,
+    branch: string
+  ): Promise<SkillSource[]> {
     const skills: SkillSource[] = [];
 
     try {
-      // Get repository info for default branch
       const { octokit, token } = await this.getOctokit();
-      const repoInfo = await octokit.repos.get({ owner, repo });
-      this.octokitPool.updateStats(token, repoInfo.headers);
-
-      if (repoInfo.data.archived) {
-        console.log(`  Skipping archived repo: ${owner}/${repo}`);
-        return [];
-      }
-
-      const defaultBranch = repoInfo.data.default_branch;
-
-      // Get the full tree recursively
       const tree = await octokit.git.getTree({
         owner,
         repo,
-        tree_sha: defaultBranch,
+        tree_sha: branch,
         recursive: 'true',
       });
       this.octokitPool.updateStats(token, tree.headers);
 
-      // Find all instruction files (SKILL.md, .cursorrules, .windsurfrules, AGENTS.md, etc.)
+      if (tree.data.truncated) {
+        // Truncated — skip this branch silently
+        return [];
+      }
+
       const instructionFiles = tree.data.tree.filter(
         (item) => {
           if (item.type !== 'blob' || !item.path) return false;
@@ -78,12 +223,80 @@ export class DeepScanCrawler {
           owner,
           repo,
           path: skillPath,
-          branch: defaultBranch,
+          branch,
           sourceFormat: matchedPattern.format,
         });
       }
+    } catch (error) {
+      if (this.isNotFoundError(error)) return [];
+      // Rate limit and other errors propagate to scanRepository()
+      throw error;
+    }
+
+    return skills;
+  }
+
+  /**
+   * Deep scan a repository for all instruction files using Git Trees API.
+   * Scans multiple important branches (version, release, well-known names).
+   * Deduplicates skills across branches, preferring the default branch.
+   */
+  async scanRepository(owner: string, repo: string, options: DeepScanOptions = {}): Promise<SkillSource[]> {
+    try {
+      const { octokit, token } = await this.getOctokit();
+      const repoInfo = await octokit.repos.get({ owner, repo });
+      this.octokitPool.updateStats(token, repoInfo.headers);
+
+      if (repoInfo.data.archived) {
+        console.log(`  Skipping archived repo: ${owner}/${repo}`);
+        return [];
+      }
+
+      const defaultBranch = repoInfo.data.default_branch;
+
+      // Determine which branches to scan
+      let branchesToScan: string[];
+      if (options.allBranches) {
+        branchesToScan = await this.listAllBranches(owner, repo, defaultBranch);
+      } else {
+        branchesToScan = await this.listImportantBranches(
+          owner, repo, defaultBranch, options.extraBranchPatterns
+        );
+      }
+
+      if (branchesToScan.length > 1) {
+        console.log(`  Scanning ${branchesToScan.length} branches: ${branchesToScan.join(', ')}`);
+      }
+
+      // Collect skills from each branch, deduplicating by path::format
+      const skillsByKey = new Map<string, SkillSource>();
+
+      for (const branch of branchesToScan) {
+        const branchSkills = await this.scanSingleBranch(owner, repo, branch);
+
+        for (const skill of branchSkills) {
+          const key = `${skill.path}::${skill.sourceFormat || 'skill.md'}`;
+          const existing = skillsByKey.get(key);
+
+          if (!existing) {
+            skillsByKey.set(key, skill);
+          } else if (existing.branch !== defaultBranch && skill.branch === defaultBranch) {
+            // Prefer default branch version
+            skillsByKey.set(key, skill);
+          }
+          // Otherwise keep existing (first non-default wins if default doesn't have it)
+        }
+      }
+
+      const skills = Array.from(skillsByKey.values());
+
       if (skills.length > 0) {
-        console.log(`  Found ${skills.length} skills in ${owner}/${repo}`);
+        const nonDefault = skills.filter(s => s.branch !== defaultBranch);
+        if (nonDefault.length > 0) {
+          console.log(`  Found ${skills.length} skills in ${owner}/${repo} (${nonDefault.length} on non-default branches)`);
+        } else {
+          console.log(`  Found ${skills.length} skills in ${owner}/${repo}`);
+        }
       }
 
       return skills;
@@ -92,15 +305,13 @@ export class DeepScanCrawler {
         return [];
       }
       if (this.isTruncatedError(error)) {
-        // Tree is truncated (>100k files), fall back to directory listing
         console.log(`  Repository ${owner}/${repo} is too large, using fallback scan`);
-        return this.fallbackScan(owner, repo);
+        return this.fallbackScanWithDefaultBranch(owner, repo);
       }
       if (this.isRateLimitError(error)) {
         console.log(`  Rate limit hit scanning ${owner}/${repo}, waiting for token rotation...`);
         await this.tokenManager.checkAndRotate();
-        // Retry once after rotation
-        return this.scanRepository(owner, repo);
+        return this.scanRepository(owner, repo, options);
       }
       throw error;
     }
@@ -114,9 +325,27 @@ export class DeepScanCrawler {
   }
 
   /**
-   * Fallback scan for large repositories - scan known skill directories
+   * Wrapper: fetches defaultBranch then runs fallback scan.
+   * Used when getTree is truncated for repos with >100k files.
    */
-  private async fallbackScan(owner: string, repo: string): Promise<SkillSource[]> {
+  private async fallbackScanWithDefaultBranch(owner: string, repo: string): Promise<SkillSource[]> {
+    let defaultBranch = 'main';
+    try {
+      const { octokit, token } = await this.getOctokit();
+      const repoInfo = await octokit.repos.get({ owner, repo });
+      this.octokitPool.updateStats(token, repoInfo.headers);
+      defaultBranch = repoInfo.data.default_branch;
+    } catch {
+      // Use 'main' as safe fallback
+    }
+    return this.fallbackScan(owner, repo, defaultBranch);
+  }
+
+  /**
+   * Fallback scan for large repositories - scan known skill directories.
+   * Uses the actual defaultBranch instead of hardcoding 'main'.
+   */
+  private async fallbackScan(owner: string, repo: string, defaultBranch: string): Promise<SkillSource[]> {
     const skills: SkillSource[] = [];
     const knownPaths = ['skills', '.claude/skills', '.github/skills', '.codex/skills', ''];
 
@@ -128,6 +357,7 @@ export class DeepScanCrawler {
           owner,
           repo,
           path: basePath || '.',
+          ref: defaultBranch,
         });
 
         this.octokitPool.updateStats(token, response.headers);
@@ -141,7 +371,7 @@ export class DeepScanCrawler {
             owner,
             repo,
             path: basePath || '.',
-            branch: 'main',
+            branch: defaultBranch,
             sourceFormat: 'skill.md' as SourceFormat,
           });
         }
@@ -155,6 +385,7 @@ export class DeepScanCrawler {
               owner,
               repo,
               path: dir.path,
+              ref: defaultBranch,
             });
             this.octokitPool.updateStats(subToken, subDir.headers);
 
@@ -165,7 +396,7 @@ export class DeepScanCrawler {
                   owner,
                   repo,
                   path: dir.path,
-                  branch: 'main',
+                  branch: defaultBranch,
                   sourceFormat: 'skill.md' as SourceFormat,
                 });
               }
@@ -190,7 +421,7 @@ export class DeepScanCrawler {
             owner,
             repo,
             path: '.',
-            branch: 'main',
+            branch: defaultBranch,
             sourceFormat: pattern.format,
           });
         }
@@ -206,14 +437,15 @@ export class DeepScanCrawler {
    * Scan multiple repositories
    */
   async scanRepositories(
-    repos: Array<{ owner: string; repo: string }>
+    repos: Array<{ owner: string; repo: string }>,
+    options: DeepScanOptions = {}
   ): Promise<Map<string, SkillSource[]>> {
     const results = new Map<string, SkillSource[]>();
 
     for (const { owner, repo } of repos) {
       try {
         console.log(`Deep scanning: ${owner}/${repo}`);
-        const skills = await this.scanRepository(owner, repo);
+        const skills = await this.scanRepository(owner, repo, options);
         results.set(`${owner}/${repo}`, skills);
       } catch (error) {
         console.warn(`  Failed to scan ${owner}/${repo}:`, error);

services/indexer/src/strategies/index.ts

@@ -7,6 +7,7 @@ import { createAwesomeListCrawler } from './awesome-list.js';
 import { createTopicSearchCrawler } from './topic-search.js';
 import { createDeepScanCrawler } from './deep-scan.js';
 import { createForkNetworkCrawler } from './fork-network.js';
+import { createCommitsSearchCrawler } from './commits-search.js';
 import { TokenManager } from '../token-manager.js';
 import type { SkillSource } from 'skillhub-core';
 
@@ -41,6 +42,7 @@ export class StrategyOrchestrator {
   private topicCrawler: ReturnType<typeof createTopicSearchCrawler>;
   private deepScanCrawler: ReturnType<typeof createDeepScanCrawler>;
   private forkCrawler: ReturnType<typeof createForkNetworkCrawler>;
+  private commitsCrawler: ReturnType<typeof createCommitsSearchCrawler>;
   private tokenManager: TokenManager;
 
   constructor(tokenManager?: TokenManager) {
@@ -49,6 +51,7 @@ export class StrategyOrchestrator {
     this.topicCrawler = createTopicSearchCrawler(this.tokenManager);
     this.deepScanCrawler = createDeepScanCrawler(this.tokenManager);
     this.forkCrawler = createForkNetworkCrawler(this.tokenManager);
+    this.commitsCrawler = createCommitsSearchCrawler(this.tokenManager);
   }
 
   /**
@@ -169,6 +172,32 @@ export class StrategyOrchestrator {
     };
   }
 
+  /**
+   * Run commits search discovery (finds repos with recent SKILL.md commits)
+   */
+  async runCommitsSearchStrategy(): Promise<DiscoveryResult> {
+    console.log('\n=== Running Commits Search Strategy ===');
+    const startTime = Date.now();
+
+    const repoResults = await this.commitsCrawler.discoverReposFromCommits(30);
+
+    const repos = repoResults.map((r) => ({
+      owner: r.owner,
+      repo: r.repo,
+      stars: r.stars,
+      discoveredVia: 'commits-search',
+    }));
+
+    console.log(`Commits search strategy completed in ${Date.now() - startTime}ms`);
+    console.log(`Discovered ${repos.length} repositories`);
+
+    return {
+      source: 'commits-search',
+      repos,
+      skills: [],
+    };
+  }
+
   /**
    * Run all strategies and return combined results
    */
@@ -230,7 +259,24 @@ export class StrategyOrchestrator {
       console.error('Topic search strategy failed:', error);
     }
 
-    // 3. Fork network (for high-star repos from above)
+    // 3. Commits search (find repos with recent SKILL.md commits)
+    try {
+      const commitsResult = await this.runCommitsSearchStrategy();
+      stats.byStrategy['commits-search'] = {
+        repos: commitsResult.repos.length,
+        skills: 0,
+      };
+      for (const repo of commitsResult.repos) {
+        const key = `${repo.owner}/${repo.repo}`.toLowerCase();
+        if (!allRepos.has(key)) {
+          allRepos.set(key, repo);
+        }
+      }
+    } catch (error) {
+      console.error('Commits search strategy failed:', error);
+    }
+
+    // 4. Fork network (for high-star repos from above)
     try {
       const highStarRepos = Array.from(allRepos.values())
         .filter((r) => (r.stars || 0) >= 10)
@@ -283,10 +329,15 @@ export function createStrategyOrchestrator(tokenManager?: TokenManager | string)
 // Re-export strategy creators
 export { createAwesomeListCrawler } from './awesome-list.js';
 export { createTopicSearchCrawler } from './topic-search.js';
-export { createDeepScanCrawler } from './deep-scan.js';
+export { createDeepScanCrawler, filterAndSortBranches } from './deep-scan.js';
+export type { DeepScanOptions } from './deep-scan.js';
 export { createForkNetworkCrawler } from './fork-network.js';
+export { createPopularReposCrawler } from './popular-repos.js';
+export { createCommitsSearchCrawler } from './commits-search.js';
 
 // Re-export types
 export type { RepoReference } from './awesome-list.js';
 export type { RepoResult } from './topic-search.js';
 export type { ForkInfo } from './fork-network.js';
+export type { PopularRepoResult } from './popular-repos.js';
+export type { CommitSearchRepoResult } from './commits-search.js';

services/indexer/src/strategies/popular-repos.ts

@@ -0,0 +1,163 @@
+import type { Octokit } from '@octokit/rest';
+import { TokenManager } from '../token-manager.js';
+import { OctokitPool } from '../octokit-pool.js';
+
+export interface PopularRepoResult {
+  owner: string;
+  repo: string;
+  stars: number;
+  forks: number;
+  defaultBranch: string;
+  isArchived: boolean;
+}
+
+/**
+ * Popular Repos Discovery Strategy
+ * Discovers popular GitHub repos (by stars) and adds them to discovered_repos
+ * so deep-scan can check their branches for SKILL.md files.
+ *
+ * This catches repos that add SKILL.md on non-default branches
+ * which GitHub Code Search doesn't index.
+ */
+export class PopularReposCrawler {
+  private octokitPool: OctokitPool;
+  private tokenManager: TokenManager;
+
+  constructor(tokenManager?: TokenManager) {
+    this.tokenManager = tokenManager || TokenManager.getInstance();
+    this.octokitPool = new OctokitPool(this.tokenManager);
+  }
+
+  private async getOctokit(): Promise<{ octokit: Octokit; token: string }> {
+    return this.octokitPool.getBestInstance();
+  }
+
+  /**
+   * Discover popular repos by star count ranges.
+   * Segments by star ranges to bypass GitHub's 1000-result-per-query limit.
+   */
+  async discoverPopularRepos(minStars: number = 1000): Promise<PopularRepoResult[]> {
+    const starRanges = this.buildStarRanges(minStars);
+    const allRepos = new Map<string, PopularRepoResult>();
+
+    console.log(`Discovering popular repos (${minStars}+ stars) across ${starRanges.length} ranges...`);
+
+    for (const range of starRanges) {
+      try {
+        console.log(`\n  Searching: stars:${range}`);
+        const repos = await this.searchByStarRange(range);
+        for (const repo of repos) {
+          const key = `${repo.owner}/${repo.repo}`.toLowerCase();
+          if (!allRepos.has(key)) {
+            allRepos.set(key, repo);
+          }
+        }
+        console.log(`  Found ${repos.length} repos (total unique: ${allRepos.size})`);
+      } catch (error) {
+        console.warn(`  Failed for range stars:${range}:`, error instanceof Error ? error.message : error);
+      }
+    }
+
+    console.log(`\nTotal unique repos discovered: ${allRepos.size}`);
+    return Array.from(allRepos.values());
+  }
+
+  /**
+   * Build star ranges to segment search queries.
+   * GitHub search returns max 1000 results per query.
+   */
+  private buildStarRanges(minStars: number): string[] {
+    const ranges: string[] = [];
+    const breakpoints = [500, 1000, 2000, 5000, 10000, 25000, 50000, 100000];
+
+    // Filter breakpoints based on minStars
+    const relevantBreaks = breakpoints.filter(b => b >= minStars);
+
+    for (let i = 0; i < relevantBreaks.length; i++) {
+      const low = i === 0 ? minStars : relevantBreaks[i - 1];
+      const high = relevantBreaks[i];
+      if (low < high) {
+        ranges.push(`${low}..${high}`);
+      }
+    }
+
+    // Add the final "greater than" range
+    const lastBreak = relevantBreaks[relevantBreaks.length - 1] || minStars;
+    ranges.push(`>${lastBreak}`);
+
+    return ranges;
+  }
+
+  /**
+   * Search repos within a specific star range.
+   */
+  private async searchByStarRange(starRange: string): Promise<PopularRepoResult[]> {
+    const results: PopularRepoResult[] = [];
+    const maxPages = 10;
+    const perPage = 100;
+
+    for (let page = 1; page <= maxPages; page++) {
+      try {
+        const { octokit, token } = await this.getOctokit();
+        const response = await octokit.search.repos({
+          q: `stars:${starRange}`,
+          sort: 'stars',
+          order: 'desc',
+          per_page: perPage,
+          page,
+        });
+        this.octokitPool.updateStats(token, response.headers);
+
+        for (const repo of response.data.items) {
+          if (!repo.archived) {
+            results.push({
+              owner: repo.owner.login,
+              repo: repo.name,
+              stars: repo.stargazers_count,
+              forks: repo.forks_count,
+              defaultBranch: repo.default_branch,
+              isArchived: repo.archived,
+            });
+          }
+        }
+
+        if (response.data.items.length < perPage) break;
+      } catch (error) {
+        if (this.isBeyondResultsLimit(error)) {
+          console.log(`    Reached 1000-result limit for stars:${starRange}`);
+          break;
+        }
+        if (this.isRateLimitError(error)) {
+          console.log(`    Rate limit hit, rotating token...`);
+          await this.tokenManager.checkAndRotate();
+          page--; // Retry same page
+          continue;
+        }
+        throw error;
+      }
+    }
+
+    return results;
+  }
+
+  private isRateLimitError(error: unknown): boolean {
+    if (!(error instanceof Error)) return false;
+    const status = (error as { status?: number }).status;
+    return status === 403 || status === 429;
+  }
+
+  private isBeyondResultsLimit(error: unknown): boolean {
+    return (
+      error instanceof Error &&
+      'status' in error &&
+      (error as { status: number }).status === 422
+    );
+  }
+}
+
+export function createPopularReposCrawler(tokenManager?: TokenManager | string): PopularReposCrawler {
+  if (typeof tokenManager === 'string') {
+    return new PopularReposCrawler(new TokenManager([tokenManager]));
+  }
+  return new PopularReposCrawler(tokenManager);
+}

services/indexer/vitest.config.ts

@@ -0,0 +1,9 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    globals: true,
+    environment: 'node',
+    include: ['src/**/*.test.ts'],
+  },
+});