diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index da30e4b37..8db7a4ac6 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -45,6 +45,9 @@ jobs: working-directory: ./framework run: just docs-build + - name: Copy robots.txt + run: cp framework/docs/src/robots.txt framework/docs/book/ + - name: Generate sitemap run: | cd framework/docs/book diff --git a/framework/docs/book.toml b/framework/docs/book.toml index 36bcae90a..bda3a4d8b 100644 --- a/framework/docs/book.toml +++ b/framework/docs/book.toml @@ -1,7 +1,6 @@ [book] authors = ["cyberhoward"] language = "en" -multilingual = false src = "src" title = "Abstract Money" @@ -32,3 +31,7 @@ git-repository-url = "https://github.com/AbstractSDK/abstract/tree/main/framew no-section-label = true preferred-dark-theme = "abstract" # [output.linkcheck] + +[output.llms-txt] + +[output.llms-txt-full] diff --git a/framework/docs/src/robots.txt b/framework/docs/src/robots.txt new file mode 100644 index 000000000..7b1b9a24b --- /dev/null +++ b/framework/docs/src/robots.txt @@ -0,0 +1,9 @@ +# Search crawlers - block .md files to avoid duplicate content +User-agent: Googlebot +Disallow: /*.md$ + +User-agent: Bingbot +Disallow: /*.md$ + +User-agent: * +Allow: / diff --git a/framework/justfile b/framework/justfile index c4146a9f2..232069e7b 100644 --- a/framework/justfile +++ b/framework/justfile @@ -81,8 +81,17 @@ docs-serve *FLAGS: docs-build: (cd docs && mdbook build) + # Copy llms.txt files to html directory + cp docs/book/llms-txt/llms.txt docs/book/html/ + cp docs/book/llms-txt-full/llms-full.txt docs/book/html/ + # Copy source markdown files for AI agents (excluding SUMMARY.md) + cd docs && find src -name "*.md" ! -name "SUMMARY.md" -exec sh -c 'mkdir -p "book/html/$(dirname "${1#src/}")" && cp "$1" "book/html/${1#src/}"' _ {} \; + # Move all content from html/ to book/ root and clean up subdirectories + cp -r docs/book/html/. docs/book/ + rm -rf docs/book/html docs/book/llms-txt docs/book/llms-txt-full docs-install: cargo install mdbook --vers "0.4.28" --locked cargo install mdbook-mermaid --vers "0.12.6" --locked cargo install mdbook-admonish --vers "1.15.0" --locked + cargo install mdbook-llms-txt-tools --vers "0.1.1" --locked