docs(llms): generate llms.txt from content (#3246)

docs/modules/llms/module.ts

@@ -0,0 +1,47 @@
+import { addPrerenderRoutes, addServerScanDir, createResolver, defineNuxtModule, useLogger } from '@nuxt/kit'
+import type { SQLOperator } from '@nuxt/content'
+
+export interface ModuleOptions {
+  domain: string
+  sections: Array<{
+    title: string
+    collection: string
+    description?: string
+    filters?: Array<{
+      field: string
+      operator: SQLOperator
+      value?: string
+    }>
+  }>
+  title?: string
+  description?: string
+  notes?: string[]
+}
+
+export default defineNuxtModule({
+  meta: {
+    name: 'llms',
+    configKey: 'llms'
+  },
+  setup(options, nuxt) {
+    const { resolve } = createResolver(import.meta.url)
+    const logger = useLogger('llms')
+
+    nuxt.options.runtimeConfig.llms = {
+      domain: options.domain,
+      title: options.title,
+      description: options.description,
+      notes: options.notes,
+      sections: options.sections || [{ title: 'Docs', collection: 'content' }]
+    }
+
+    if (!options.domain) {
+      logger.warn('Please provide a domain for the LLMs module. LLMS docs require a domain to be set.')
+    }
+
+    addServerScanDir(resolve('runtime/server'))
+
+    addPrerenderRoutes('/llms.txt')
+    addPrerenderRoutes('/llms_full.txt')
+  }
+})

docs/modules/llms/runtime/server/routes/llms.txt.get.ts

@@ -0,0 +1,56 @@
+import { joinURL } from 'ufo'
+import type { ModuleOptions } from '~~/modules/llms/module'
+
+export default eventHandler(async (event) => {
+  const options = useRuntimeConfig(event).llms as ModuleOptions
+
+  const llms = [
+    `# ${options.title || 'Documentation'}`
+  ]
+
+  if (options.description) {
+    llms.push(`> ${options.description}`)
+  }
+
+  llms.push(
+    '## Documentation Sets',
+    `- [Complete Documentation](${joinURL(options.domain, '/llms_full.txt')}): The complete documentation including all content`
+  )
+
+  for (const section of options.sections) {
+    // @ts-expect-error - typecheck does not derect server querryCollection
+    const query = queryCollection(event, section.collection)
+      .select('path', 'title', 'description')
+      .where('path', 'NOT LIKE', '%/.navigation')
+
+    if (section.filters) {
+      for (const filter of section.filters) {
+        query.where(filter.field, filter.operator, filter.value)
+      }
+    }
+
+    const docs = await query.all()
+
+    const links = docs.map((doc) => {
+      return `- [${doc.title}](${joinURL(options.domain, doc.path)}): ${doc.description}`
+    })
+
+    llms.push(`## ${section.title}`)
+
+    if (section.description) {
+      llms.push(section.description)
+    }
+
+    llms.push(links.join('\n'))
+  }
+
+  if (options.notes && options.notes.length) {
+    llms.push(
+      '## Notes',
+      (options.notes || []).map(note => `- ${note}`).join('\n')
+    )
+  }
+
+  setHeader(event, 'Content-Type', 'text/plain')
+  return llms.join('\n\n')
+})

docs/modules/llms/runtime/server/routes/llms_full.txt.get.ts

@@ -0,0 +1,78 @@
+import { joinURL, hasProtocol } from 'ufo'
+import type { ModuleOptions } from '~~/modules/llms/module'
+import { stringifyMarkdown } from '@nuxtjs/mdc/runtime'
+import type { MDCRoot } from '@nuxtjs/mdc'
+
+export default eventHandler(async (event) => {
+  const options = useRuntimeConfig(event).llms as ModuleOptions
+
+  const llms = []
+
+  for (const section of options.sections) {
+    // @ts-expect-error - typecheck does not derect server querryCollection
+    const query = queryCollection(event, section.collection)
+      .where('path', 'NOT LIKE', '%/.navigation')
+
+    if (section.filters) {
+      for (const filter of section.filters) {
+        query.where(filter.field, filter.operator, filter.value)
+      }
+    }
+
+    const docs = await query.all()
+
+    for (const doc of docs) {
+      let markdown = await stringifyMarkdown(decompressBody(doc.body, options), {})
+
+      if (!markdown?.trim().startsWith('# ')) {
+        markdown = `# ${doc.title}\n\n${markdown}`
+      }
+      llms.push(markdown)
+    }
+  }
+
+  if (options.notes && options.notes.length) {
+    llms.push(
+      '## Notes',
+      (options.notes || []).map(note => `- ${note}`).join('\n')
+    )
+  }
+
+  setHeader(event, 'Content-Type', 'text/plain')
+  return llms.join('\n\n')
+})
+
+// decompress utils is part of Content module and not exposed yet
+// We can refactor this after exposing the utils
+function decompressBody(body: any, options: ModuleOptions): MDCRoot {
+  const linkProps = ['href', 'src', 'to']
+
+  function decompressNode(input: any) {
+    if (typeof input === 'string') {
+      return {
+        type: 'text',
+        value: input
+      }
+    }
+
+    const [tag, props, ...children] = input
+
+    for (const prop of linkProps) {
+      if (props[prop] && !hasProtocol(props[prop])) {
+        props[prop] = joinURL(options.domain, props[prop])
+      }
+    }
+
+    return {
+      type: 'element',
+      tag,
+      props,
+      children: children.map(decompressNode)
+    }
+  }
+
+  return {
+    type: 'root',
+    children: body.value.map(decompressNode)
+  }
+}

docs/modules/llms/runtime/server/tsconfig.json

@@ -0,0 +1,3 @@
+{
+	"extends": "../../../../.nuxt/tsconfig.server.json"
+}

docs/nuxt.config.ts

@@ -22,7 +22,8 @@ export default defineNuxtConfig({
       nuxt.hook('components:dirs', (dirs) => {
         dirs.unshift({ path: resolve('./app/components/content/examples'), pathPrefix: false, prefix: '', global: true })
       })
-    }
+    },
+    '~~/modules/llms/module'
   ],
 
   app: {
@@ -147,6 +148,50 @@ export default defineNuxtConfig({
   image: {
     provider: 'ipx'
   },
+  llms: {
+    domain: 'https://ui3.nuxt.dev',
+    title: 'Nuxt UI v3',
+    description: 'A comprehensive, Nuxt-integrated UI library providing a rich set of fully-styled, accessible and highly customizable components for building modern web applications.',
+    sections: [
+      {
+        title: 'Getting Started',
+        collection: 'content',
+        filters: [
+          {
+            field: 'path',
+            operator: 'LIKE',
+            value: '/getting-started%'
+          }
+        ]
+      },
+      {
+        title: 'Components',
+        collection: 'content',
+        filters: [
+          {
+            field: 'path',
+            operator: 'LIKE',
+            value: '/components/%'
+          }
+        ]
+      },
+      {
+        title: 'Composables',
+        collection: 'content',
+        filters: [
+          {
+            field: 'path',
+            operator: 'LIKE',
+            value: '/composables/%'
+          }
+        ]
+      }
+    ],
+    notes: [
+      'The documentation excludes Nuxt UI v2 content.',
+      'The content is automatically generated from the same source as the official documentation.'
+    ]
+  },
 
   uiPro: {
     license: 'oss'