Initial release v1.0.0

Open-source marketplace for AI Agent skills.

Features:
- Next.js 15 web app with i18n (en/fa)
- CLI tool for skill installation (npx skillhub)
- GitHub crawler/indexer with multi-strategy discovery
- Security scanning for all indexed skills
- Self-hostable with Docker

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

apps/web/app/robots.ts

@@ -0,0 +1,37 @@
+import type { MetadataRoute } from 'next';
+
+/**
+ * Dynamic robots.txt for mirror server support
+ *
+ * Primary server (IS_PRIMARY_SERVER=true): Allow all crawlers
+ * Mirror server (IS_PRIMARY_SERVER=false): Block all crawlers to prevent duplicate content
+ */
+export default function robots(): MetadataRoute.Robots {
+  const isPrimary = process.env.IS_PRIMARY_SERVER !== 'false';
+  const primaryDomain = process.env.NEXT_PUBLIC_APP_URL || 'https://skills.palebluedot.live';
+
+  if (isPrimary) {
+    return {
+      rules: [
+        {
+          userAgent: '*',
+          allow: '/',
+          disallow: ['/api/', '/monitoring/'],
+        },
+      ],
+      sitemap: `${primaryDomain}/sitemap.xml`,
+    };
+  }
+
+  // Mirror server: Block all crawlers to prevent duplicate content indexing
+  return {
+    rules: [
+      {
+        userAgent: '*',
+        disallow: '/',
+      },
+    ],
+    // Point to primary sitemap even on mirror
+    sitemap: `${primaryDomain}/sitemap.xml`,
+  };
+}