From 6dd607728b7b2700a9c55b26c2e7edb9d831bbb1 Mon Sep 17 00:00:00 2001
From: Paul Wackerow <54227730+wackerow@users.noreply.github.com>
Date: Thu, 15 Dec 2022 05:01:13 +0100
Subject: [PATCH] Add sitemap and robots.txt (#163)

* yarn add next-sitemap and config

* add sitemap to .gitignore

* commit generated robots.txt
---
 .gitignore             |  3 +++
 next-sitemap.config.js |  6 ++++++
 package.json           |  2 ++
 public/robots.txt      |  9 +++++++++
 yarn.lock              | 13 +++++++++++++
 5 files changed, 33 insertions(+)
 create mode 100644 next-sitemap.config.js
 create mode 100644 public/robots.txt

diff --git a/.gitignore b/.gitignore
index c87c9b392c..0ad2e445b8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -34,3 +34,6 @@ yarn-error.log*
 # typescript
 *.tsbuildinfo
 next-env.d.ts
+
+# Sitemaps
+sitemap*.xml
diff --git a/next-sitemap.config.js b/next-sitemap.config.js
new file mode 100644
index 0000000000..8cd3c222d2
--- /dev/null
+++ b/next-sitemap.config.js
@@ -0,0 +1,6 @@
+/** @type {import('next-sitemap').IConfig} */
+
+module.exports = {
+  siteUrl: 'https://geth.ethereum.org',
+  generateRobotsTxt: true
+};
diff --git a/package.json b/package.json
index 124a93d3d1..4f142c8ece 100644
--- a/package.json
+++ b/package.json
@@ -7,6 +7,7 @@
   "scripts": {
     "dev": "next dev",
     "build": "npm run lint && next build && npm run format:fix",
+    "postbuild": "next-sitemap",
     "start": "next start",
     "lint": "next lint",
     "format:fix": "prettier . --write --config .prettierrc --ignore-path .prettierignore --loglevel warn"
@@ -23,6 +24,7 @@
     "gray-matter": "^4.0.3",
     "js-yaml": "^4.1.0",
     "next": "^13.0.2",
+    "next-sitemap": "^3.1.32",
     "react": "18.2.0",
     "react-dom": "18.2.0",
     "react-markdown": "^8.0.3",
diff --git a/public/robots.txt b/public/robots.txt
new file mode 100644
index 0000000000..2883f50af7
--- /dev/null
+++ b/public/robots.txt
@@ -0,0 +1,9 @@
+# *
+User-agent: *
+Allow: /
+
+# Host
+Host: https://geth.ethereum.org
+
+# Sitemaps
+Sitemap: https://geth.ethereum.org/sitemap.xml
diff --git a/yarn.lock b/yarn.lock
index 070259b380..d8d9f0bd0d 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -806,6 +806,11 @@
   resolved "https://registry.yarnpkg.com/@chakra-ui/visually-hidden/-/visually-hidden-2.0.9.tgz#b43a3dd0bc1108954ad0eeb50d0261887ab5e31c"
   integrity sha512-PkNxrRGp9H3bdqEaoo8XGt/AL9UuGRTom0/9XJa+G/Dj8Cy1sDuamOWk3pN/ZQs46RokfK9Uh5LqPY5dwSDweg==
 
+"@corex/deepmerge@^4.0.29":
+  version "4.0.29"
+  resolved "https://registry.yarnpkg.com/@corex/deepmerge/-/deepmerge-4.0.29.tgz#af9debf07d7f6b0d2a9d04a266abf2c1418ed2f6"
+  integrity sha512-q/yVUnqckA8Do+EvAfpy7RLdumnBy9ZsducMUtZTvpdbJC7azEf1hGtnYYxm0QfphYxjwggv6XtH64prvS1W+A==
+
 "@ctrl/tinycolor@^3.4.0":
   version "3.4.1"
   resolved "https://registry.npmjs.org/@ctrl/tinycolor/-/tinycolor-3.4.1.tgz"
@@ -3215,6 +3220,14 @@ natural-compare@^1.4.0:
   resolved "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz"
   integrity sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==
 
+next-sitemap@^3.1.32:
+  version "3.1.32"
+  resolved "https://registry.yarnpkg.com/next-sitemap/-/next-sitemap-3.1.32.tgz#e4a7227cab23b5e5c68bc54d335b86d3ff1e05f8"
+  integrity sha512-jkIKpwLXpWWTPfmDO46+6nu4+qpar4CjvUwCR9rYZHWtzE/wFfaCVFKpGtFMl6MFjpu8GjiE6kWFEa7uF3bzzg==
+  dependencies:
+    "@corex/deepmerge" "^4.0.29"
+    minimist "^1.2.6"
+
 next@^13.0.2:
   version "13.0.2"
   resolved "https://registry.npmjs.org/next/-/next-13.0.2.tgz"