fix: add optional llms.txt generation (#74)

add llms.txt generation

Author: tylerdave

docs/Configuration.md

@@ -115,6 +115,17 @@ Atom feed generation. Requires `site.base_url` to be set.
 | `modified_date_fallback` | `bool`      | `false`                                    | Use file modification time when no frontmatter date field matches    |
 | `timezone`               | `str`       | `"UTC"`                                    | IANA timezone for date display (e.g. `"US/Eastern"`)                 |
 
+## `[llms_txt]`
+
+Generates an `llms.txt` file following the [llmstxt.org](https://llmstxt.org/) specification, making site content discoverable by LLMs. Requires `site.base_url` to be set.
+
+| Field         | Type   | Default | Description                                       |
+| ------------- | ------ | ------- | ------------------------------------------------- |
+| `enabled`     | `bool` | `false` | Enable llms.txt generation                        |
+| `description` | `str`  | `""`    | Site summary shown as a blockquote below the title |
+
+Pages are grouped by named collection first, then by top-level directory, then root-level pages.
+
 ## `[hooks]`
 
 Shell commands executed at build lifecycle stages. Commands run sequentially; any non-zero exit aborts the build.

docs/rockgarden.toml

@@ -3,6 +3,11 @@ title = "Rockgarden"
 clean_urls = true
 output = "../_site"
 ignore_patterns = []
+base_url = "https://rockgarden.build"
+
+[llms_txt]
+enabled = true
+description = "Rockgarden is an opinionated static site generator that turns a folder of Markdown files into a fast, searchable, interlinked website. It supports CommonMark, GitHub Flavored Markdown, and Obsidian syntax out of the box."
 
 [dates]
 timezone = "US/Eastern"

src/rockgarden/config.py

@@ -138,6 +138,13 @@ class FeedConfig(BaseModel):
     collections: list[str] = Field(default_factory=list)
 
 
+class LlmsTxtConfig(BaseModel):
+    """llms.txt generation configuration."""
+
+    enabled: bool = False
+    description: str = ""
+
+
 class HooksConfig(BaseModel):
     """Build hook commands executed at lifecycle stages."""
 
@@ -169,6 +176,7 @@ class Config(BaseModel):
     search: SearchConfig = Field(default_factory=SearchConfig)
     dates: DatesConfig = Field(default_factory=DatesConfig)
     feed: FeedConfig = Field(default_factory=FeedConfig)
+    llms_txt: LlmsTxtConfig = Field(default_factory=LlmsTxtConfig)
     hooks: HooksConfig = Field(default_factory=HooksConfig)
     collections: list[CollectionConfig] = Field(default_factory=list)
 

src/rockgarden/output/builder.py

@@ -48,6 +48,7 @@
 )
 from rockgarden.output.build_info import get_build_info
 from rockgarden.output.feed import build_atom_feed
+from rockgarden.output.llms_txt import build_llms_txt
 from rockgarden.output.search import build_search_index, strip_html
 from rockgarden.output.sitemap import build_sitemap
 from rockgarden.output.tags import build_tag_pages, collect_tags
@@ -750,6 +751,21 @@ def build_site(
         feed_file.parent.mkdir(parents=True, exist_ok=True)
         feed_file.write_text(feed_xml)
 
+    # Generate llms.txt if base_url is configured and llms_txt enabled
+    if config.site.base_url and config.llms_txt.enabled:
+        llms_txt_content = build_llms_txt(
+            pages,
+            rendered_folder_indexes,
+            collections,
+            config.site.base_url,
+            config.site.title,
+            config.llms_txt.description,
+            clean_urls,
+            base_path,
+            config.nav.links,
+        )
+        (output / "llms.txt").write_text(llms_txt_content)
+
     # Generate 404 page
     not_found_template = env.get_template("404.html")
     not_found_layout = resolve_layout({}, config.theme.default_layout)

src/rockgarden/output/llms_txt.py

@@ -0,0 +1,118 @@
+"""llms.txt generation following the llmstxt.org specification."""
+
+from __future__ import annotations
+
+from rockgarden.config import NavLinkConfig
+from rockgarden.content.collection import Collection
+from rockgarden.content.models import Page
+from rockgarden.nav.folder_index import FolderIndex
+from rockgarden.urls import get_url
+
+
+def _flatten_nav_links(links: list[NavLinkConfig]) -> list[NavLinkConfig]:
+    """Flatten nested nav links, keeping only entries with a URL."""
+    result: list[NavLinkConfig] = []
+    for link in links:
+        if link.url:
+            result.append(link)
+        result.extend(_flatten_nav_links(link.children))
+    return result
+
+
+def build_llms_txt(
+    pages: list[Page],
+    folder_indexes: list[FolderIndex],
+    collections: dict[str, Collection],
+    base_url: str,
+    site_title: str,
+    description: str = "",
+    clean_urls: bool = True,
+    base_path: str = "",
+    nav_links: list[NavLinkConfig] | None = None,
+) -> str:
+    """Generate an llms.txt string.
+
+    Sections are ordered: named collections first, then remaining pages grouped
+    by top-level directory, then root-level pages, then nav links.
+
+    Args:
+        pages: All content pages.
+        folder_indexes: All folder index pages.
+        collections: Named collections mapping.
+        base_url: Site base URL (e.g., "https://example.com").
+        site_title: Site title for the H1 heading.
+        description: Site description for the blockquote.
+        clean_urls: Whether clean URLs are enabled.
+        base_path: URL path prefix.
+        nav_links: Custom navigation links from config.
+
+    Returns:
+        llms.txt content string.
+    """
+    lines: list[str] = [f"# {site_title}"]
+
+    if description:
+        lines.append("")
+        lines.append(f"> {description}")
+
+    collected_slugs: set[str] = set()
+    for col in collections.values():
+        entries = [e for e in col.entries if isinstance(e, Page)]
+        if not entries:
+            continue
+        lines.append("")
+        lines.append(f"## {col.name}")
+        lines.append("")
+        for entry in sorted(entries, key=lambda p: p.title.lower()):
+            url = base_url + get_url(entry.slug, clean_urls, base_path)
+            lines.append(f"- [{entry.title}]({url})")
+            collected_slugs.add(entry.slug)
+
+    dir_groups: dict[str, list[Page | FolderIndex]] = {}
+    root_items: list[Page | FolderIndex] = []
+
+    all_items: list[Page | FolderIndex] = [
+        p for p in pages if p.slug not in collected_slugs
+    ]
+    collected_dirs = {col.config.source for col in collections.values() if col.entries}
+    all_items.extend(
+        fi for fi in folder_indexes if fi.slug.rsplit("/", 1)[0] not in collected_dirs
+    )
+
+    for item in all_items:
+        parts = item.slug.split("/")
+        if len(parts) > 1:
+            top_dir = parts[0]
+            dir_groups.setdefault(top_dir, []).append(item)
+        else:
+            root_items.append(item)
+
+    for dir_name in sorted(dir_groups):
+        items = sorted(dir_groups[dir_name], key=lambda p: p.title.lower())
+        lines.append("")
+        lines.append(f"## {dir_name}")
+        lines.append("")
+        for item in items:
+            url = base_url + get_url(item.slug, clean_urls, base_path)
+            lines.append(f"- [{item.title}]({url})")
+
+    if root_items:
+        items = sorted(root_items, key=lambda p: p.title.lower())
+        lines.append("")
+        lines.append("## Pages")
+        lines.append("")
+        for item in items:
+            url = base_url + get_url(item.slug, clean_urls, base_path)
+            lines.append(f"- [{item.title}]({url})")
+
+    if nav_links:
+        flat_links = _flatten_nav_links(nav_links)
+        if flat_links:
+            lines.append("")
+            lines.append("## Links")
+            lines.append("")
+            for link in flat_links:
+                lines.append(f"- [{link.label}]({link.url})")
+
+    lines.append("")
+    return "\n".join(lines)

tests/test_llms_txt.py

@@ -0,0 +1,179 @@
+"""Tests for llms.txt generation."""
+
+from pathlib import Path
+
+from rockgarden.config import CollectionConfig, NavLinkConfig
+from rockgarden.content.collection import Collection
+from rockgarden.content.models import Page
+from rockgarden.nav.folder_index import FolderIndex
+from rockgarden.output.llms_txt import build_llms_txt
+
+
+class TestBuildLlmsTxt:
+    def _make_page(self, slug, title=None, source_prefix=""):
+        source = f"{source_prefix}/{slug}.md" if source_prefix else f"{slug}.md"
+        return Page(
+            source_path=Path(source),
+            slug=slug,
+            frontmatter={"title": title} if title else {},
+        )
+
+    def test_title_heading(self):
+        result = build_llms_txt([], [], {}, "https://example.com", "My Site")
+        assert result.startswith("# My Site\n")
+
+    def test_description_blockquote(self):
+        result = build_llms_txt(
+            [], [], {}, "https://example.com", "My Site", description="A cool site."
+        )
+        assert "> A cool site." in result
+
+    def test_no_description(self):
+        result = build_llms_txt([], [], {}, "https://example.com", "My Site")
+        assert ">" not in result
+
+    def test_root_pages_under_pages_heading(self):
+        pages = [self._make_page("about", "About")]
+        result = build_llms_txt(pages, [], {}, "https://example.com", "My Site")
+        assert "## Pages" in result
+        assert "- [About](https://example.com/about/)" in result
+
+    def test_pages_grouped_by_directory(self):
+        pages = [
+            self._make_page("docs/intro", "Introduction"),
+            self._make_page("docs/setup", "Setup"),
+        ]
+        result = build_llms_txt(pages, [], {}, "https://example.com", "My Site")
+        assert "## docs" in result
+        assert "- [Introduction](https://example.com/docs/intro/)" in result
+        assert "- [Setup](https://example.com/docs/setup/)" in result
+        assert "## Pages" not in result
+
+    def test_collection_pages_grouped_separately(self):
+        page = self._make_page("blog/post1", "First Post", source_prefix="blog")
+        col_config = CollectionConfig(name="Blog", source="blog")
+        col = Collection(name="Blog", config=col_config, entries=[page])
+        result = build_llms_txt(
+            [page], [], {"blog": col}, "https://example.com", "My Site"
+        )
+        assert "## Blog" in result
+        assert "- [First Post](https://example.com/blog/post1/)" in result
+        # Should not also appear under directory grouping
+        lines = result.split("\n")
+        assert lines.count("## Blog") == 1
+        assert "## blog" not in result
+
+    def test_folder_indexes_included(self):
+        folders = [FolderIndex(slug="docs/index", title="Docs", children=[])]
+        result = build_llms_txt([], folders, {}, "https://example.com", "My Site")
+        assert "## docs" in result
+        assert "- [Docs](https://example.com/docs/)" in result
+
+    def test_clean_urls_false(self):
+        pages = [self._make_page("about", "About")]
+        result = build_llms_txt(
+            pages, [], {}, "https://example.com", "My Site", clean_urls=False
+        )
+        assert "https://example.com/about.html" in result
+
+    def test_base_path(self):
+        pages = [self._make_page("about", "About")]
+        result = build_llms_txt(
+            pages, [], {}, "https://example.com", "My Site", base_path="/docs"
+        )
+        assert "https://example.com/docs/about/" in result
+
+    def test_alphabetical_ordering(self):
+        pages = [
+            self._make_page("zebra", "Zebra"),
+            self._make_page("apple", "Apple"),
+        ]
+        result = build_llms_txt(pages, [], {}, "https://example.com", "My Site")
+        apple_pos = result.index("Apple")
+        zebra_pos = result.index("Zebra")
+        assert apple_pos < zebra_pos
+
+    def test_collections_before_directories(self):
+        page_col = self._make_page("blog/post", "Post", source_prefix="blog")
+        page_dir = self._make_page("docs/guide", "Guide")
+        col_config = CollectionConfig(name="Blog", source="blog")
+        col = Collection(name="Blog", config=col_config, entries=[page_col])
+        result = build_llms_txt(
+            [page_col, page_dir],
+            [],
+            {"blog": col},
+            "https://example.com",
+            "My Site",
+        )
+        blog_pos = result.index("## Blog")
+        docs_pos = result.index("## docs")
+        assert blog_pos < docs_pos
+
+    def test_empty_site(self):
+        result = build_llms_txt([], [], {}, "https://example.com", "My Site")
+        assert result == "# My Site\n"
+
+    def test_nav_links_section(self):
+        links = [
+            NavLinkConfig(label="GitHub", url="https://github.com/example"),
+            NavLinkConfig(label="Tags", url="/tags/"),
+        ]
+        result = build_llms_txt(
+            [], [], {}, "https://example.com", "My Site", nav_links=links
+        )
+        assert "## Links" in result
+        assert "- [GitHub](https://github.com/example)" in result
+        assert "- [Tags](/tags/)" in result
+
+    def test_nav_links_after_pages(self):
+        links = [NavLinkConfig(label="GitHub", url="https://github.com/example")]
+        pages = [self._make_page("about", "About")]
+        result = build_llms_txt(
+            pages, [], {}, "https://example.com", "My Site", nav_links=links
+        )
+        pages_pos = result.index("## Pages")
+        links_pos = result.index("## Links")
+        assert pages_pos < links_pos
+
+    def test_nav_links_flattens_nested(self):
+        links = [
+            NavLinkConfig(
+                label="Parent",
+                url="",
+                children=[
+                    NavLinkConfig(label="Child", url="https://example.com/child"),
+                ],
+            ),
+        ]
+        result = build_llms_txt(
+            [], [], {}, "https://example.com", "My Site", nav_links=links
+        )
+        assert "- [Child](https://example.com/child)" in result
+        assert "Parent" not in result
+
+    def test_no_nav_links_no_section(self):
+        result = build_llms_txt(
+            [], [], {}, "https://example.com", "My Site", nav_links=[]
+        )
+        assert "## Links" not in result
+
+    def test_collection_folder_index_not_duplicated(self):
+        page = self._make_page("blog/post1", "First Post", source_prefix="blog")
+        folder = FolderIndex(slug="blog/index", title="Blog", children=[])
+        col_config = CollectionConfig(name="Blog", source="blog")
+        col = Collection(name="Blog", config=col_config, entries=[page])
+        result = build_llms_txt(
+            [page], [folder], {"blog": col}, "https://example.com", "My Site"
+        )
+        assert "## Blog" in result
+        assert "## blog" not in result
+
+    def test_trailing_newline(self):
+        result = build_llms_txt(
+            [self._make_page("about", "About")],
+            [],
+            {},
+            "https://example.com",
+            "My Site",
+        )
+        assert result.endswith("\n")