97b427831a
Open-source marketplace for AI Agent skills. Features: - Next.js 15 web app with i18n (en/fa) - CLI tool for skill installation (npx skillhub) - GitHub crawler/indexer with multi-strategy discovery - Security scanning for all indexed skills - Self-hostable with Docker Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
38 lines
975 B
TypeScript
38 lines
975 B
TypeScript
import type { MetadataRoute } from 'next';
|
|
|
|
/**
|
|
* Dynamic robots.txt for mirror server support
|
|
*
|
|
* Primary server (IS_PRIMARY_SERVER=true): Allow all crawlers
|
|
* Mirror server (IS_PRIMARY_SERVER=false): Block all crawlers to prevent duplicate content
|
|
*/
|
|
export default function robots(): MetadataRoute.Robots {
|
|
const isPrimary = process.env.IS_PRIMARY_SERVER !== 'false';
|
|
const primaryDomain = process.env.NEXT_PUBLIC_APP_URL || 'https://skills.palebluedot.live';
|
|
|
|
if (isPrimary) {
|
|
return {
|
|
rules: [
|
|
{
|
|
userAgent: '*',
|
|
allow: '/',
|
|
disallow: ['/api/', '/monitoring/'],
|
|
},
|
|
],
|
|
sitemap: `${primaryDomain}/sitemap.xml`,
|
|
};
|
|
}
|
|
|
|
// Mirror server: Block all crawlers to prevent duplicate content indexing
|
|
return {
|
|
rules: [
|
|
{
|
|
userAgent: '*',
|
|
disallow: '/',
|
|
},
|
|
],
|
|
// Point to primary sitemap even on mirror
|
|
sitemap: `${primaryDomain}/sitemap.xml`,
|
|
};
|
|
}
|