From 57d787f5ba854aba5ea8c2b6888b8bb4d7ce6910 Mon Sep 17 00:00:00 2001 From: bherbruck Date: Sun, 25 Oct 2026 18:35:56 -0400 Subject: [PATCH] Docs site: canonical links, robots.txt and sitemap.xml (#6888) Search Console lists docs.solidcraft.io/, /introduction.html (the same page) and /guide/mcp.html as duplicates without a user-selected canonical. The Pages build now adds a canonical link to every book page (index.html and introduction.html both name /), skips the redirect stubs that already have one, marks 404 and the toc frame noindex, and writes a robots.txt pointing at a generated sitemap.xml. Co-authored-by: Claude Opus 5.5 --- .github/workflows/pages.yml | 1 + docs/book/seo.sh | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100755 docs/book/seo.sh diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 9b9a69d49..02c2c0200 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -65,6 +65,7 @@ jobs: run: | mkdir -p site cp -r target/book/. site/ + bash docs/book/seo.sh site # canonical links, robots.txt, sitemap.xml cp -r dist/web site/app - uses: actions/configure-pages@v5 id: pages diff --git a/docs/book/seo.sh b/docs/book/seo.sh new file mode 100755 index 000000000..748bc7bef --- /dev/null +++ b/docs/book/seo.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Adds to every built book page and writes robots.txt and sitemap.xml. +# mdBook's templates cannot turn a chapter's .md path into its .html URL, so this runs on the built +# tree (usage: seo.sh ). index.html and introduction.html are the same page: both point at /. +set -euo pipefail +dir="${1:?usage: seo.sh }" +base="https://docs.solidcraft.io" +cd "$dir" +urls=() +while IFS= read -r f; do + f="${f#./}" + case "$f" in + 404.html|toc.html) # not content pages: keep them out of the index + sed -i 's|||' "$f"; continue ;; + esac + grep -q 'rel="canonical"' "$f" && continue # mdBook's redirect stubs already name their target + case "$f" in + index.html|introduction.html) path="" ;; + */index.html) path="${f%index.html}" ;; + *) path="$f" ;; + esac + url="$base/$path" + sed -i "s|||" "$f" + [ "$f" = print.html ] || [ "$f" = introduction.html ] || urls+=("$url") +done < <(find . -name '*.html' | sort) +{ + echo '' + echo '' + for u in "${urls[@]}"; do echo " $u"; done + echo '' +} > sitemap.xml +printf 'User-agent: *\nAllow: /\n\nSitemap: %s/sitemap.xml\n' "$base" > robots.txt