fix: resolve CSRF race condition + sync updates

- fetchWithCsrf now awaits token initialization before sending requests
- Sync crawler and worker updates

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

.github/workflows/ci.yml

@@ -1,150 +0,0 @@
-name: CI
-
-on:
-  push:
-    branches: [main]
-  pull_request:
-    branches: [main]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  lint:
-    name: Lint
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: pnpm/action-setup@v4
-
-      - uses: actions/setup-node@v4
-        with:
-          node-version: '20'
-          cache: 'pnpm'
-
-      - name: Install dependencies
-        run: pnpm install --frozen-lockfile
-
-      - name: Lint and Type check
-        run: pnpm turbo lint typecheck
-
-  test:
-    name: Unit Tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: pnpm/action-setup@v4
-
-      - uses: actions/setup-node@v4
-        with:
-          node-version: '20'
-          cache: 'pnpm'
-
-      - name: Install dependencies
-        run: pnpm install --frozen-lockfile
-
-      - name: Run core tests
-        run: pnpm --filter @skillhub/core test:run
-
-      - name: Run db tests
-        run: pnpm --filter @skillhub/db test:run
-
-      - name: Run web API tests
-        run: pnpm --filter @skillhub/web test:run
-
-      - name: Run CLI tests
-        run: pnpm --filter @skillhub/cli test:run
-
-  e2e:
-    name: E2E Tests
-    runs-on: ubuntu-latest
-    needs: [build]
-    # TODO: Enable when CI has database/redis services
-    if: false
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: pnpm/action-setup@v4
-
-      - uses: actions/setup-node@v4
-        with:
-          node-version: '20'
-          cache: 'pnpm'
-
-      - name: Install dependencies
-        run: pnpm install --frozen-lockfile
-
-      - name: Install Playwright browsers
-        run: pnpm --filter @skillhub/web exec playwright install --with-deps chromium
-
-      - name: Build web app and dependencies
-        run: pnpm turbo build --filter=@skillhub/web
-
-      - name: Run E2E tests
-        run: pnpm --filter @skillhub/web test:e2e --project=chromium
-        env:
-          PLAYWRIGHT_BASE_URL: http://localhost:3000
-
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    needs: [lint, test]
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: pnpm/action-setup@v4
-
-      - uses: actions/setup-node@v4
-        with:
-          node-version: '20'
-          cache: 'pnpm'
-
-      - name: Install dependencies
-        run: pnpm install --frozen-lockfile
-
-      - name: Build
-        run: pnpm build
-
-  docker:
-    name: Build Docker Images
-    runs-on: ubuntu-latest
-    needs: [build]
-    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build Web image
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          file: ./apps/web/Dockerfile
-          push: false
-          tags: ghcr.io/${{ github.repository_owner }}/skillhub-web:latest
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-      - name: Build Indexer image
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          file: ./services/indexer/Dockerfile
-          push: false
-          tags: ghcr.io/${{ github.repository_owner }}/skillhub-indexer:latest
-          cache-from: type=gha
-          cache-to: type=gha,mode=max

apps/web/lib/csrf-client.ts

@@ -14,6 +14,9 @@ const CSRF_HEADER_NAME = 'x-csrf-token';
 // Store the CSRF token in memory (will be populated on first page load)
 let csrfToken: string | null = null;
 
+// Pending initialization promise (deduplicates concurrent init calls)
+let initPromise: Promise<string | null> | null = null;
+
 /**
  * Set the CSRF token (called when receiving a response with the token)
  */
@@ -43,6 +46,20 @@ export function createHeadersWithCsrf(additionalHeaders?: HeadersInit): Headers
   return headers;
 }
 
+/**
+ * Ensure the CSRF token is available, initializing if needed.
+ * Deduplicates concurrent calls so only one GET /api/health fires.
+ */
+async function ensureCsrfToken(): Promise<string | null> {
+  if (csrfToken) return csrfToken;
+  if (!initPromise) {
+    initPromise = initializeCsrfToken().finally(() => {
+      initPromise = null;
+    });
+  }
+  return initPromise;
+}
+
 /**
  * Create a fetch wrapper that automatically includes the CSRF token
  * and extracts new tokens from responses
@@ -51,6 +68,9 @@ export async function fetchWithCsrf(
   url: string | URL,
   options?: RequestInit
 ): Promise<Response> {
+  // Wait for token to be ready before sending a protected request
+  await ensureCsrfToken();
+
   const token = getCsrfToken();
 
   const headers = new Headers(options?.headers);

services/indexer/src/crawl.ts

@@ -7,7 +7,7 @@
 import { INSTRUCTION_FILE_PATTERNS, type SourceFormat } from 'skillhub-core';
 import { scheduleFullCrawl, scheduleIncrementalCrawl, getQueueStats, getQueue } from './queue.js';
 import { syncAllSkillsToMeilisearch, checkMeilisearchHealth } from './meilisearch-sync.js';
-import { createDb, skillQueries, categoryQueries, discoveredRepoQueries, awesomeListQueries, addRequestQueries, userQueries } from '@skillhub/db';
+import { createDb, skillQueries, categoryQueries, discoveredRepoQueries, awesomeListQueries, addRequestQueries, userQueries, sql } from '@skillhub/db';
 import { createStrategyOrchestrator, createDeepScanCrawler, createAwesomeListCrawler } from './strategies/index.js';
 import { createCrawler } from './crawler.js';
 import { indexSkill } from './skill-indexer.js';

services/indexer/src/worker.ts

@@ -1,7 +1,7 @@
 import type { Job } from 'bullmq';
 import { Worker } from 'bullmq';
 import pLimit from 'p-limit';
-import { createDb, type Database, discoveredRepoQueries, awesomeListQueries, addRequestQueries, skillQueries } from '@skillhub/db';
+import { createDb, type Database, discoveredRepoQueries, awesomeListQueries, addRequestQueries, skillQueries, sql } from '@skillhub/db';
 import { GitHubCrawler, createCrawler } from './crawler.js';
 import type { IndexJobData, IndexJobResult } from './queue.js';
 import { setupRecurringJobs } from './queue.js';