Add documentation markdown generation

.github/workflows/gh-pages.yml

@@ -45,6 +45,9 @@ jobs:
         working-directory: ./framework
         run: just docs-build
 
+      - name: Copy robots.txt
+        run: cp framework/docs/src/robots.txt framework/docs/book/
+
       - name: Generate sitemap
         run: |
           cd framework/docs/book

framework/docs/book.toml

@@ -1,7 +1,6 @@
 [book]
 authors      = ["cyberhoward"]
 language     = "en"
-multilingual = false
 src          = "src"
 title        = "Abstract Money"
 
@@ -32,3 +31,7 @@ git-repository-url   = "https://github.com/AbstractSDK/abstract/tree/main/framew
 no-section-label     = true
 preferred-dark-theme = "abstract"
 # [output.linkcheck]
+
+[output.llms-txt]
+
+[output.llms-txt-full]

framework/docs/src/robots.txt

@@ -0,0 +1,9 @@
+# Search crawlers - block .md files to avoid duplicate content
+User-agent: Googlebot
+Disallow: /*.md$
+
+User-agent: Bingbot
+Disallow: /*.md$
+
+User-agent: *
+Allow: /

framework/justfile

@@ -81,8 +81,17 @@ docs-serve *FLAGS:
 
 docs-build:
   (cd docs && mdbook build)
+  # Copy llms.txt files to html directory
+  cp docs/book/llms-txt/llms.txt docs/book/html/
+  cp docs/book/llms-txt-full/llms-full.txt docs/book/html/
+  # Copy source markdown files for AI agents (excluding SUMMARY.md)
+  cd docs && find src -name "*.md" ! -name "SUMMARY.md" -exec sh -c 'mkdir -p "book/html/$(dirname "${1#src/}")" && cp "$1" "book/html/${1#src/}"' _ {} \;
+  # Move all content from html/ to book/ root and clean up subdirectories
+  cp -r docs/book/html/. docs/book/
+  rm -rf docs/book/html docs/book/llms-txt docs/book/llms-txt-full
 
 docs-install:
   cargo install mdbook --vers "0.4.28" --locked
   cargo install mdbook-mermaid --vers "0.12.6" --locked
   cargo install mdbook-admonish --vers "1.15.0" --locked
+  cargo install mdbook-llms-txt-tools --vers "0.1.1" --locked