Add markdown serving for AI-readable documentation

.github/workflows/pages.yml

@@ -0,0 +1,54 @@
+name: Build and Deploy Jekyll Site
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.1'
+          bundler-cache: true
+
+      - name: Setup Pages
+        id: pages
+        uses: actions/configure-pages@v5
+
+      - name: Build with Jekyll
+        run: bundle exec jekyll build --baseurl "${{ steps.pages.outputs.base_path }}"
+        env:
+          JEKYLL_ENV: production
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    if: github.ref == 'refs/heads/main'
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.gitignore

@@ -1,2 +1,7 @@
 _site/
 *.dic
+Gemfile.lock
+.bundle/
+vendor/
+.jekyll-cache/
+.jekyll-metadata

.wordlist.txt

@@ -4,11 +4,13 @@ AST
 Aldis
 Analytics
 APIs
+baseurl
 BFS
 BLAS
 Benchmarking
 Booleans
 booleans
+bundler
 CLI
 Cmd
 CMD
@@ -40,6 +42,7 @@ FalkorDBGraph
 FalkorDBLite
 FalkorDBQAChain
 Gadepally
+Gemfile
 Gemini
 Geospatial
 GETUSER

AI_READABLE.md

@@ -0,0 +1,60 @@
+# AI-Readable Documentation
+
+This documentation site is designed to be AI-friendly by providing clean markdown versions of all documentation pages.
+
+## Features
+
+### 1. Markdown URL Endpoints
+
+For every documentation page, a markdown version is available by appending `.md` to the URL.
+
+**Examples:**
+- HTML: `https://docs.falkordb.com/` → Markdown: `https://docs.falkordb.com/.md`
+- HTML: `https://docs.falkordb.com/getting-started/clients` → Markdown: `https://docs.falkordb.com/getting-started/clients.md`
+- HTML: `https://docs.falkordb.com/cypher/indexing/vector-index` → Markdown: `https://docs.falkordb.com/cypher/indexing/vector-index.md`
+
+### 2. Clean Markdown Output
+
+The markdown versions are stripped of:
+- Navigation menus
+- Sidebars
+- Footers
+- Theme elements
+- JavaScript and styling
+
+Only the core documentation content is included, making it optimal for AI tools to parse and understand.
+
+### 3. Content Structure
+
+Each markdown page includes:
+- Page title (as H1)
+- Page description (if available)
+- Main content body
+
+## Implementation Details
+
+### Jekyll Plugin
+
+A custom Jekyll plugin (`_plugins/markdown_generator.rb`) automatically generates `.md` versions of all documentation pages during the build process.
+
+### Clean Layout
+
+The `_layouts/markdown.html` layout renders content without any theme-specific elements, providing clean, focused markdown output.
+
+### Build Process
+
+The site uses a custom GitHub Actions workflow (`.github/workflows/pages.yml`) that:
+1. Builds the Jekyll site with custom plugins enabled
+2. Generates both HTML and markdown versions
+3. Deploys to GitHub Pages
+
+## For AI Tools
+
+AI tools can:
+1. Directly request the `.md` version of any page
+2. Parse clean markdown without HTML noise
+3. Access structured documentation content efficiently
+
+## Note on Content Negotiation
+
+While the Accept header approach (`Accept: text/markdown`) would be ideal, GitHub Pages' static hosting doesn't support server-side content negotiation. The `.md` URL suffix provides a reliable alternative that works with all clients.

Gemfile

@@ -0,0 +1,17 @@
+source "https://rubygems.org"
+
+gem "jekyll", "~> 4.3"
+gem "just-the-docs", "~> 0.10"
+
+group :jekyll_plugins do
+  gem "jekyll-sitemap"
+  gem "jekyll-redirect-from"
+end
+
+platforms :mingw, :x64_mingw, :mswin, :jruby do
+  gem "tzinfo", ">= 1", "< 3"
+  gem "tzinfo-data"
+end
+
+gem "wdm", "~> 0.1", :install_if => Gem.win_platform?
+gem "http_parser.rb", "~> 0.6.0", :platforms => [:jruby]

_config.yml

@@ -1,4 +1,4 @@
-remote_theme: just-the-docs/just-the-docs
+theme: just-the-docs
 title: FalkorDB Docs
 description: The FalkorDB documentation
 logo: "/images/falkor-logo.png"
@@ -27,7 +27,7 @@ plugins:
   - jekyll-redirect-from
 
 # Footer last edited timestamp
-last_edit_timestamp: true 
+last_edit_timestamp: true
 last_edit_time_format: "%b %e %Y at %I:%M %p" # uses ruby's time format: https://ruby-doc.org/stdlib-2.7.0/libdoc/time/rdoc/Time.html
 
 # Footer "Edit this page on GitHub" link text
@@ -36,3 +36,12 @@ gh_edit_link_text: "Edit this page on GitHub."
 gh_edit_repository: "https://github.com/FalkorDB/docs/"
 gh_edit_branch: "main"
 gh_edit_view_mode: "edit"
+
+# Exclude files from processing
+exclude:
+  - Gemfile
+  - Gemfile.lock
+  - node_modules
+  - vendor
+  - .bundle
+  - README.md

_layouts/markdown.html

@@ -0,0 +1,10 @@
+---
+# Clean markdown layout for AI tools
+# Strips navigation, footer, and other non-content elements
+# Outputs plain text content type for markdown
+---
+{% if page.title %}# {{ page.title }}
+
+{% endif %}{% if page.description %}{{ page.description }}
+
+{% endif %}{{ content | strip }}

_plugins/markdown_generator.rb

@@ -0,0 +1,62 @@
+# Jekyll plugin to generate .md versions of documentation pages for AI tools
+# This creates clean markdown files at URL.md for each documentation page
+
+module Jekyll
+  class MarkdownPageGenerator < Generator
+    safe true
+    priority :lowest
+
+    def generate(site)
+      # Collect all pages that should have .md versions
+      pages_to_process = []
+
+      site.pages.each do |page|
+        # Only process markdown source files
+        next unless page.name.end_with?('.md')
+
+        # Skip if already a .md URL endpoint
+        next if page.url.end_with?('.md')
+
+        # Skip special files
+        next if page.name.start_with?('_')
+
+        pages_to_process << page
+      end
+
+      # Generate .md versions
+      pages_to_process.each do |source_page|
+        md_page = MarkdownPage.new(site, source_page)
+        site.pages << md_page
+      end
+    end
+  end
+
+  class MarkdownPage < Page
+    def initialize(site, source_page)
+      @site = site
+      @base = site.source
+      @dir = File.dirname(source_page.relative_path)
+
+      # Generate unique name for .md version
+      base_name = File.basename(source_page.name, '.md')
+      @name = "#{base_name}.source.md"
+
+      # Process the filename
+      self.process(@name)
+
+      # Copy and modify data
+      self.data = {}
+      self.data['layout'] = 'markdown'
+      self.data['title'] = source_page.data['title']
+      self.data['description'] = source_page.data['description']
+
+      # Set permalink to be original_url.md
+      base_url = source_page.url.chomp('/')
+      base_url = '/' if base_url.empty?
+      self.data['permalink'] = "#{base_url}.md"
+
+      # Get the content
+      self.content = source_page.content
+    end
+  end
+end