sync: stale skill detection, sentry filtering, claim removal, review improvements

Stale Detection: - Detect skills removed/moved from GitHub (3 consecutive 404s threshold) - Hide stale skills from browse/search, show warning banner on detail page - Serve cached files with isStale flag when GitHub returns 404 - Add stale-check crawler command for batch verification - CLI shows warning when installing stale skills from cache Sentry & Error Handling: - Filter browser extension errors and add denyUrls - Anti-inflation measures and sentinel recalibration for curation Claim & Removal: - Enhanced ClaimForm with repo-level removal support - Add repo-removal-request API endpoint with tests - Improved owner page with bilingual content Review Pipeline: - Review version and reviewer tracking in submit API - Source format filter for pending reviews - Updated review tests Other: - Updated i18n strings (en/fa) - BrowseFilters improvements - Dockerfile updates Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-12 05:59:26 +03:30
parent 447f9eff59
commit 4212b5ba47
31 changed files with 1590 additions and 58 deletions
--- a/packages/db/src/queries.test.ts
+++ b/packages/db/src/queries.test.ts
@@ -6,6 +6,7 @@ import {
  ratingQueries,
  installationQueries,
  favoriteQueries,
+  discoveredRepoQueries,
 } from './queries.js';
 import {
  createTestSkill,
@@ -22,13 +23,14 @@ import {

 // Mock the schema imports
 vi.mock('./schema.js', () => ({
-  skills: { id: 'id', name: 'name', description: 'description', githubStars: 'github_stars', downloadCount: 'download_count', rating: 'rating', updatedAt: 'updated_at', isFeatured: 'is_featured', isVerified: 'is_verified', securityScore: 'security_score', viewCount: 'view_count', ratingCount: 'rating_count', ratingSum: 'rating_sum' },
+  skills: { id: 'id', name: 'name', description: 'description', githubStars: 'github_stars', downloadCount: 'download_count', rating: 'rating', updatedAt: 'updated_at', isFeatured: 'is_featured', isVerified: 'is_verified', securityScore: 'security_score', viewCount: 'view_count', ratingCount: 'rating_count', ratingSum: 'rating_sum', githubOwner: 'github_owner', githubRepo: 'github_repo', isBlocked: 'is_blocked', isOwnerClaimed: 'is_owner_claimed', isDuplicate: 'is_duplicate' },
  categories: { id: 'id', name: 'name', slug: 'slug', sortOrder: 'sort_order', skillCount: 'skill_count' },
  skillCategories: { skillId: 'skill_id', categoryId: 'category_id' },
  users: { id: 'id', githubId: 'github_id', username: 'username', avatarUrl: 'avatar_url' },
  ratings: { id: 'id', skillId: 'skill_id', userId: 'user_id', rating: 'rating', createdAt: 'created_at' },
  installations: { id: 'id', skillId: 'skill_id', platform: 'platform', method: 'method' },
  favorites: { userId: 'user_id', skillId: 'skill_id', createdAt: 'created_at' },
+  discoveredRepos: { id: 'id', isBlocked: 'is_blocked', isArchived: 'is_archived', lastScanned: 'last_scanned', githubStars: 'github_stars', discoveredVia: 'discovered_via', owner: 'owner', repo: 'repo', githubForks: 'github_forks', defaultBranch: 'default_branch', hasSkillMd: 'has_skill_md', skillCount: 'skill_count', scanError: 'scan_error', updatedAt: 'updated_at' },
 }));

 // Helper to create a chainable mock
@@ -659,3 +661,127 @@ describe('favoriteQueries', () => {
    });
  });
 });
+
+describe('skillQueries (repo block/unblock)', () => {
+  describe('blockByRepo', () => {
+    it('should call update with isBlocked: true for matching owner/repo', async () => {
+      const mockDb = createMockDb();
+
+      await skillQueries.blockByRepo(mockDb as any, 'rawveg', 'skillsforge-marketplace');
+
+      expect(mockDb.update).toHaveBeenCalled();
+    });
+  });
+
+  describe('unblockByRepo', () => {
+    it('should call update with isBlocked: false for matching owner/repo', async () => {
+      const mockDb = createMockDb();
+
+      await skillQueries.unblockByRepo(mockDb as any, 'rawveg', 'skillsforge-marketplace');
+
+      expect(mockDb.update).toHaveBeenCalled();
+    });
+  });
+
+  describe('setOwnerClaimed', () => {
+    it('should call update with isOwnerClaimed: true for matching owner/repo', async () => {
+      const mockDb = createMockDb();
+
+      await skillQueries.setOwnerClaimed(mockDb as any, 'rawveg', 'skillsforge-marketplace');
+
+      expect(mockDb.update).toHaveBeenCalled();
+    });
+  });
+
+  describe('countByRepo', () => {
+    it('should return count of non-blocked skills for owner/repo', async () => {
+      const mockDb = createMockDb([{ count: 16 }]);
+
+      const result = await skillQueries.countByRepo(mockDb as any, 'rawveg', 'skillsforge-marketplace');
+
+      expect(result).toBe(16);
+      expect(mockDb.select).toHaveBeenCalled();
+    });
+
+    it('should return 0 when no matching skills', async () => {
+      const mockDb = createMockDb([{ count: 0 }]);
+
+      const result = await skillQueries.countByRepo(mockDb as any, 'unknown', 'unknown-repo');
+
+      expect(result).toBe(0);
+    });
+
+    it('should return 0 when result is empty', async () => {
+      const mockDb = createMockDb([]);
+
+      const result = await skillQueries.countByRepo(mockDb as any, 'owner', 'repo');
+
+      expect(result).toBe(0);
+    });
+  });
+});
+
+describe('discoveredRepoQueries', () => {
+  describe('blockRepo', () => {
+    it('should call update with isBlocked: true for the repo id', async () => {
+      const mockDb = createMockDb();
+
+      await discoveredRepoQueries.blockRepo(mockDb as any, 'rawveg/skillsforge-marketplace');
+
+      expect(mockDb.update).toHaveBeenCalled();
+    });
+  });
+
+  describe('unblockRepo', () => {
+    it('should call update with isBlocked: false for the repo id', async () => {
+      const mockDb = createMockDb();
+
+      await discoveredRepoQueries.unblockRepo(mockDb as any, 'rawveg/skillsforge-marketplace');
+
+      expect(mockDb.update).toHaveBeenCalled();
+    });
+  });
+
+  describe('getNeedingScanning', () => {
+    it('should return repos that need scanning', async () => {
+      const repos = [
+        { id: 'owner/repo1', isBlocked: false },
+        { id: 'owner/repo2', isBlocked: false },
+      ];
+      const mockDb = createMockDb(repos);
+
+      const result = await discoveredRepoQueries.getNeedingScanning(mockDb as any);
+
+      expect(result).toEqual(repos);
+      expect(mockDb.select).toHaveBeenCalled();
+    });
+
+    it('should return empty array when no repos need scanning', async () => {
+      const mockDb = createMockDb([]);
+
+      const result = await discoveredRepoQueries.getNeedingScanning(mockDb as any);
+
+      expect(result).toEqual([]);
+    });
+  });
+
+  describe('getById', () => {
+    it('should return repo when found', async () => {
+      const repo = { id: 'rawveg/skillsforge-marketplace', isBlocked: true };
+      const mockDb = createMockDb([repo]);
+
+      const result = await discoveredRepoQueries.getById(mockDb as any, 'rawveg/skillsforge-marketplace');
+
+      expect(result).toEqual(repo);
+      expect(mockDb.select).toHaveBeenCalled();
+    });
+
+    it('should return null when repo not found', async () => {
+      const mockDb = createMockDb([]);
+
+      const result = await discoveredRepoQueries.getById(mockDb as any, 'nonexistent/repo');
+
+      expect(result).toBeNull();
+    });
+  });
+});
--- a/packages/db/src/queries.ts
+++ b/packages/db/src/queries.ts
@@ -36,7 +36,7 @@ type DB = PostgresJsDatabase<typeof schema>;
 * Skills with skill_type=NULL (newly crawled, not yet curated) are included
 * so they don't disappear between curate.mjs runs.
 */
-const browseReadyFilter = sql`(${skills.isDuplicate} = false)`;
+const browseReadyFilter = sql`(${skills.isDuplicate} = false OR ${skills.isOwnerClaimed} = true) AND ${skills.isStale} = false`;

 /** Minimum repo age gap (in days) to trust repo_created_at over stars for duplicate detection */
 const REPO_AGE_THRESHOLD_DAYS = 75;
@@ -688,6 +688,10 @@ export const skillQueries = {
            : (skill.securityStatus && skill.qualityScore != null)
              ? { reviewStatus: sql`CASE WHEN ${skills.reviewStatus} IS NULL OR ${skills.reviewStatus} = 'unreviewed' THEN 'auto-scored' ELSE ${skills.reviewStatus} END` }
              : {}),
+          // Clear stale state on successful re-index (skill is accessible again)
+          isStale: false,
+          staleSince: null,
+          staleCheckCount: 0,
        },
      })
      .returning();
@@ -759,6 +763,44 @@ export const skillQueries = {
    await db.update(skills).set({ isBlocked: false }).where(eq(skills.id, id));
  },

+  /**
+   * Mark all skills from a repo as owner-claimed (exempt from duplicate hiding)
+   */
+  setOwnerClaimed: async (db: DB, owner: string, repo: string) => {
+    await db.update(skills)
+      .set({ isOwnerClaimed: true })
+      .where(and(eq(skills.githubOwner, owner), eq(skills.githubRepo, repo)));
+  },
+
+  /**
+   * Block all skills from a repo (owner requested repo-level removal)
+   */
+  blockByRepo: async (db: DB, owner: string, repo: string) => {
+    await db.update(skills)
+      .set({ isBlocked: true })
+      .where(and(eq(skills.githubOwner, owner), eq(skills.githubRepo, repo)));
+  },
+
+  /**
+   * Unblock all skills from a repo (owner re-enabled their repository)
+   */
+  unblockByRepo: async (db: DB, owner: string, repo: string) => {
+    await db.update(skills)
+      .set({ isBlocked: false })
+      .where(and(eq(skills.githubOwner, owner), eq(skills.githubRepo, repo)));
+  },
+
+  /**
+   * Count non-blocked skills in a repo
+   */
+  countByRepo: async (db: DB, owner: string, repo: string): Promise<number> => {
+    const result = await db
+      .select({ count: sql<number>`count(*)::int` })
+      .from(skills)
+      .where(and(eq(skills.githubOwner, owner), eq(skills.githubRepo, repo), eq(skills.isBlocked, false)));
+    return result[0]?.count ?? 0;
+  },
+
  /**
   * Check if a skill is blocked
   */
@@ -771,6 +813,92 @@ export const skillQueries = {
    return result[0]?.isBlocked ?? false;
  },

+  /**
+   * Increment stale check count. After 3 consecutive 404s, mark skill as stale.
+   * Sets stale_since on first detection. Idempotent if skill doesn't exist.
+   */
+  incrementStaleCheck: async (db: DB, id: string) => {
+    const STALE_THRESHOLD = 3;
+
+    const existing = await db
+      .select({
+        staleCheckCount: skills.staleCheckCount,
+        staleSince: skills.staleSince,
+      })
+      .from(skills)
+      .where(eq(skills.id, id))
+      .limit(1);
+
+    if (!existing[0]) return;
+
+    const newCount = (existing[0].staleCheckCount ?? 0) + 1;
+    const isNowStale = newCount >= STALE_THRESHOLD;
+
+    await db.update(skills).set({
+      staleCheckCount: newCount,
+      staleSince: existing[0].staleSince ?? new Date(),
+      isStale: isNowStale,
+    }).where(eq(skills.id, id));
+  },
+
+  /**
+   * Clear stale state when a skill is successfully re-fetched from GitHub
+   */
+  clearStaleState: async (db: DB, id: string) => {
+    await db.update(skills).set({
+      isStale: false,
+      staleSince: null,
+      staleCheckCount: 0,
+    }).where(eq(skills.id, id));
+  },
+
+  /**
+   * Get stale skills (for admin/reporting)
+   */
+  getStaleSkills: async (db: DB, limit = 100, offset = 0) => {
+    return db
+      .select()
+      .from(skills)
+      .where(eq(skills.isStale, true))
+      .orderBy(desc(skills.staleSince))
+      .limit(limit)
+      .offset(offset);
+  },
+
+  /**
+   * Count stale skills
+   */
+  countStale: async (db: DB): Promise<number> => {
+    const result = await db
+      .select({ count: sql<number>`count(*)::int` })
+      .from(skills)
+      .where(eq(skills.isStale, true));
+    return result[0]?.count ?? 0;
+  },
+
+  /**
+   * Get skills to check for staleness (ordered by oldest scanned first)
+   */
+  getSkillsToStaleCheck: async (db: DB, limit = 500) => {
+    return db
+      .select({
+        id: skills.id,
+        githubOwner: skills.githubOwner,
+        githubRepo: skills.githubRepo,
+        skillPath: skills.skillPath,
+        branch: skills.branch,
+        sourceFormat: skills.sourceFormat,
+        staleCheckCount: skills.staleCheckCount,
+      })
+      .from(skills)
+      .where(and(
+        eq(skills.isBlocked, false),
+        eq(skills.isStale, false),
+      ))
+      .orderBy(asc(skills.lastScanned))
+      .limit(limit);
+  },
+
  /**
   * Get cached files for a skill
   * Returns null if cache doesn't exist or is stale (commitSha mismatch)
@@ -1266,6 +1394,7 @@ export async function runPostCrawlCuration(db: DB): Promise<{ classified: number
    UPDATE skills s SET is_duplicate = true
    WHERE s.is_blocked = false
      AND s.is_duplicate = false
+      AND s.is_owner_claimed = false
      AND s.content_hash IS NOT NULL
      AND EXISTS (
        SELECT 1 FROM skills s2
@@ -1623,6 +1752,7 @@ export const discoveredRepoQueries = {
      conditions.push(sql`${discoveredRepos.lastScanned} IS NULL`);
    }
    conditions.push(eq(discoveredRepos.isArchived, false));
+    conditions.push(eq(discoveredRepos.isBlocked, false));

    return db
      .select()
@@ -1632,6 +1762,20 @@ export const discoveredRepoQueries = {
      .limit(limit);
  },

+  /**
+   * Block a repo from re-indexing (owner requested removal of all skills)
+   */
+  blockRepo: async (db: DB, id: string) => {
+    await db.update(discoveredRepos).set({ isBlocked: true }).where(eq(discoveredRepos.id, id));
+  },
+
+  /**
+   * Unblock a repo (owner re-enabled it after previously removing it)
+   */
+  unblockRepo: async (db: DB, id: string) => {
+    await db.update(discoveredRepos).set({ isBlocked: false }).where(eq(discoveredRepos.id, id));
+  },
+
  /**
   * Mark a repo as scanned
   */
@@ -2511,6 +2655,7 @@ export const skillReviewQueries = {
      priorityReReview?: boolean;
      reReviewAll?: boolean;
      ownerLimit?: number;
+      currentReviewVersion?: number;
    } = {}
  ) => {
    const {
@@ -2521,6 +2666,7 @@ export const skillReviewQueries = {
      priorityReReview = false,
      reReviewAll = false,
      ownerLimit = 0,
+      currentReviewVersion = 0,
    } = options;

    const conditions = [
@@ -2539,6 +2685,16 @@ export const skillReviewQueries = {
      conditions.push(
        sql`${skills.reviewStatus} IN ('ai-reviewed', 'needs-re-review', 'auto-scored')`
      );
+      // Skip skills already reviewed at the current version
+      if (currentReviewVersion > 0) {
+        conditions.push(
+          sql`NOT EXISTS (
+            SELECT 1 FROM skill_reviews sr
+            WHERE sr.skill_id = ${skills.id}
+            AND sr.review_version >= ${currentReviewVersion}
+          )`
+        );
+      }
    } else if (priorityReReview) {
      conditions.push(eq(skills.reviewStatus, 'needs-re-review'));
    } else {
@@ -2570,6 +2726,12 @@ export const skillReviewQueries = {
      branch: skills.branch,
    };

+    // Re-review-all: prioritize already-reviewed skills first, then auto-scored
+    // Use sql.raw() for the ORDER BY to avoid parameterization issues in db.execute()
+    const orderBySql = reReviewAll
+      ? sql`CASE WHEN review_status IN ('ai-reviewed', 'needs-re-review') THEN 0 ELSE 1 END, quality_score DESC NULLS LAST, github_stars DESC NULLS LAST`
+      : sql`quality_score DESC NULLS LAST, github_stars DESC NULLS LAST`;
+
    // Owner-capped batch: limit skills per github_owner for diversity
    if (ownerLimit > 0) {
      const whereClause = and(...conditions);
@@ -2580,7 +2742,7 @@ export const skillReviewQueries = {
                 github_owner, github_repo, skill_path, branch,
                 ROW_NUMBER() OVER (
                   PARTITION BY github_owner
-                   ORDER BY quality_score DESC NULLS LAST, github_stars DESC NULLS LAST
+                   ORDER BY ${orderBySql}
                 ) AS owner_rank
          FROM skills
          WHERE ${whereClause}
@@ -2593,18 +2755,23 @@ export const skillReviewQueries = {
               skill_path AS "skillPath", branch
        FROM ranked
        WHERE owner_rank <= ${ownerLimit}
-        ORDER BY quality_score DESC NULLS LAST, github_stars DESC NULLS LAST
+        ORDER BY ${orderBySql}
        LIMIT ${batchSize}
        OFFSET ${offset}
      `);
      return [...results];
    }

+    // Build order: re-review priority first (if applicable), then quality
+    const orderClauses = reReviewAll
+      ? [sql`CASE WHEN ${skills.reviewStatus} IN ('ai-reviewed', 'needs-re-review') THEN 0 ELSE 1 END`, desc(skills.qualityScore), desc(skills.githubStars)]
+      : [desc(skills.qualityScore), desc(skills.githubStars)];
+
    return db
      .select(selectFields)
      .from(skills)
      .where(and(...conditions))
-      .orderBy(desc(skills.qualityScore), desc(skills.githubStars))
+      .orderBy(...orderClauses)
      .limit(batchSize)
      .offset(offset);
  },
--- a/packages/db/src/schema.ts
+++ b/packages/db/src/schema.ts
@@ -67,6 +67,9 @@ export const skills = pgTable(
    isFeatured: boolean('is_featured').default(false),
    isBlocked: boolean('is_blocked').default(false), // Blocked from re-indexing (owner requested removal)
    isDeprecated: boolean('is_deprecated').default(false), // Auto-detected from raw_content (DEPRECATED/ARCHIVED markers)
+    isStale: boolean('is_stale').default(false), // Skill files no longer accessible on GitHub (confirmed after 3 consecutive 404s)
+    staleSince: timestamp('stale_since'), // When staleness was first detected
+    staleCheckCount: integer('stale_check_count').default(0), // Consecutive 404 count (threshold: 3)
    lastScanned: timestamp('last_scanned'),

    // Curation (populated by batch scripts, not crawler)
@@ -79,6 +82,7 @@ export const skills = pgTable(
    }>(),
    skillType: text('skill_type').$type<'standalone' | 'project-bound' | 'collection' | 'aggregator'>(),
    isDuplicate: boolean('is_duplicate').default(false),
+    isOwnerClaimed: boolean('is_owner_claimed').default(false), // Owner explicitly submitted via Add Skill — exempt from duplicate hiding
    canonicalSkillId: text('canonical_skill_id'), // points to the "original" if this is a duplicate
    repoSkillCount: integer('repo_skill_count'), // cached count of skills in same repo
    reviewStatus: text('review_status').default('unreviewed'), // unreviewed → auto-scored → ai-reviewed → verified (or needs-re-review)
@@ -124,6 +128,7 @@ export const skills = pgTable(
    skillTypeIdx: index('idx_skills_type').on(table.skillType),
    duplicateIdx: index('idx_skills_duplicate').on(table.isDuplicate),
    contentHashIdx: index('idx_skills_content_hash').on(table.contentHash),
+    staleIdx: index('idx_skills_stale').on(table.isStale),
  })
 );

@@ -303,6 +308,7 @@ export const discoveredRepos = pgTable(
    githubForks: integer('github_forks').default(0),
    defaultBranch: text('default_branch').default('main'),
    isArchived: boolean('is_archived').default(false),
+    isBlocked: boolean('is_blocked').default(false), // Owner requested removal — skip re-indexing
    scanError: text('scan_error'), // Last scan error if any
    createdAt: timestamp('created_at').defaultNow().notNull(),
    updatedAt: timestamp('updated_at').defaultNow().notNull(),