robots.txt config and route

config/default.yaml

@@ -90,6 +90,10 @@ instance:
   customizations:
     javascript: '' # Directly your JavaScript code (without <script> tags). Will be eval at runtime
     css: '' # Directly your CSS code (without <style> tags). Will be injected at runtime
+  # Robot.txt rules. To allow robots to crawl your instance and allow indexation of your site, remove the '/'
+  robots: |
+    User-agent: *
+    Disallow: /
 
 services:
   # Cards configuration to format video in Twitter

config/production.yaml.example

@@ -106,6 +106,10 @@ instance:
   customizations:
     javascript: '' # Directly your JavaScript code (without <script> tags). Will be eval at runtime
     css: '' # Directly your CSS code (without <style> tags). Will be injected at runtime
+  # Robot.txt rules. To allow robots to crawl your instance and allow indexation of your site, remove the '/'
+  robots: |
+    User-agent: *
+    Disallow: /
 
 services:
   # Cards configuration to format video in Twitter

server/controllers/static.ts

@@ -44,6 +44,12 @@ staticRouter.use(
   asyncMiddleware(getPreview)
 )
 
+// robots.txt service
+staticRouter.get('/robots.txt', (req: express.Request, res: express.Response) => {
+  res.type('text/plain')
+  return res.send(CONFIG.INSTANCE.ROBOTS)
+})
+
 // ---------------------------------------------------------------------------
 
 export {

server/initializers/checker.ts

@@ -29,7 +29,7 @@ function checkMissedConfig () {
     'user.video_quota',
     'cache.previews.size', 'admin.email', 'signup.enabled', 'signup.limit', 'transcoding.enabled', 'transcoding.threads',
     'instance.name', 'instance.short_description', 'instance.description', 'instance.terms', 'instance.default_client_route',
-    'instance.default_nsfw_policy',
+    'instance.default_nsfw_policy', 'instance.robots',
     'services.twitter.username', 'services.twitter.whitelisted'
   ]
   const miss: string[] = []

server/initializers/constants.ts

@@ -180,7 +180,8 @@ const CONFIG = {
     CUSTOMIZATIONS: {
       get JAVASCRIPT () { return config.get<string>('instance.customizations.javascript') },
       get CSS () { return config.get<string>('instance.customizations.css') }
-    }
+    },
+    get ROBOTS () { return config.get<string>('instance.robots') }
   },
   SERVICES: {
     TWITTER: {