docs(llms): generate llms.txt from content (#3246)

docs/modules/llms/module.ts

@@ -0,0 +1,47 @@
+import { addPrerenderRoutes, addServerScanDir, createResolver, defineNuxtModule, useLogger } from '@nuxt/kit'
+import type { SQLOperator } from '@nuxt/content'
+
+export interface ModuleOptions {
+  domain: string
+  sections: Array<{
+    title: string
+    collection: string
+    description?: string
+    filters?: Array<{
+      field: string
+      operator: SQLOperator
+      value?: string
+    }>
+  }>
+  title?: string
+  description?: string
+  notes?: string[]
+}
+
+export default defineNuxtModule({
+  meta: {
+    name: 'llms',
+    configKey: 'llms'
+  },
+  setup(options, nuxt) {
+    const { resolve } = createResolver(import.meta.url)
+    const logger = useLogger('llms')
+
+    nuxt.options.runtimeConfig.llms = {
+      domain: options.domain,
+      title: options.title,
+      description: options.description,
+      notes: options.notes,
+      sections: options.sections || [{ title: 'Docs', collection: 'content' }]
+    }
+
+    if (!options.domain) {
+      logger.warn('Please provide a domain for the LLMs module. LLMS docs require a domain to be set.')
+    }
+
+    addServerScanDir(resolve('runtime/server'))
+
+    addPrerenderRoutes('/llms.txt')
+    addPrerenderRoutes('/llms_full.txt')
+  }
+})

docs/modules/llms/runtime/server/routes/llms.txt.get.ts

@@ -0,0 +1,56 @@
+import { joinURL } from 'ufo'
+import type { ModuleOptions } from '~~/modules/llms/module'
+
+export default eventHandler(async (event) => {
+  const options = useRuntimeConfig(event).llms as ModuleOptions
+
+  const llms = [
+    `# ${options.title || 'Documentation'}`
+  ]
+
+  if (options.description) {
+    llms.push(`> ${options.description}`)
+  }
+
+  llms.push(
+    '## Documentation Sets',
+    `- [Complete Documentation](${joinURL(options.domain, '/llms_full.txt')}): The complete documentation including all content`
+  )
+
+  for (const section of options.sections) {
+    // @ts-expect-error - typecheck does not derect server querryCollection
+    const query = queryCollection(event, section.collection)
+      .select('path', 'title', 'description')
+      .where('path', 'NOT LIKE', '%/.navigation')
+
+    if (section.filters) {
+      for (const filter of section.filters) {
+        query.where(filter.field, filter.operator, filter.value)
+      }
+    }
+
+    const docs = await query.all()
+
+    const links = docs.map((doc) => {
+      return `- [${doc.title}](${joinURL(options.domain, doc.path)}): ${doc.description}`
+    })
+
+    llms.push(`## ${section.title}`)
+
+    if (section.description) {
+      llms.push(section.description)
+    }
+
+    llms.push(links.join('\n'))
+  }
+
+  if (options.notes && options.notes.length) {
+    llms.push(
+      '## Notes',
+      (options.notes || []).map(note => `- ${note}`).join('\n')
+    )
+  }
+
+  setHeader(event, 'Content-Type', 'text/plain')
+  return llms.join('\n\n')
+})

docs/modules/llms/runtime/server/routes/llms_full.txt.get.ts

@@ -0,0 +1,78 @@
+import { joinURL, hasProtocol } from 'ufo'
+import type { ModuleOptions } from '~~/modules/llms/module'
+import { stringifyMarkdown } from '@nuxtjs/mdc/runtime'
+import type { MDCRoot } from '@nuxtjs/mdc'
+
+export default eventHandler(async (event) => {
+  const options = useRuntimeConfig(event).llms as ModuleOptions
+
+  const llms = []
+
+  for (const section of options.sections) {
+    // @ts-expect-error - typecheck does not derect server querryCollection
+    const query = queryCollection(event, section.collection)
+      .where('path', 'NOT LIKE', '%/.navigation')
+
+    if (section.filters) {
+      for (const filter of section.filters) {
+        query.where(filter.field, filter.operator, filter.value)
+      }
+    }
+
+    const docs = await query.all()
+
+    for (const doc of docs) {
+      let markdown = await stringifyMarkdown(decompressBody(doc.body, options), {})
+
+      if (!markdown?.trim().startsWith('# ')) {
+        markdown = `# ${doc.title}\n\n${markdown}`
+      }
+      llms.push(markdown)
+    }
+  }
+
+  if (options.notes && options.notes.length) {
+    llms.push(
+      '## Notes',
+      (options.notes || []).map(note => `- ${note}`).join('\n')
+    )
+  }
+
+  setHeader(event, 'Content-Type', 'text/plain')
+  return llms.join('\n\n')
+})
+
+// decompress utils is part of Content module and not exposed yet
+// We can refactor this after exposing the utils
+function decompressBody(body: any, options: ModuleOptions): MDCRoot {
+  const linkProps = ['href', 'src', 'to']
+
+  function decompressNode(input: any) {
+    if (typeof input === 'string') {
+      return {
+        type: 'text',
+        value: input
+      }
+    }
+
+    const [tag, props, ...children] = input
+
+    for (const prop of linkProps) {
+      if (props[prop] && !hasProtocol(props[prop])) {
+        props[prop] = joinURL(options.domain, props[prop])
+      }
+    }
+
+    return {
+      type: 'element',
+      tag,
+      props,
+      children: children.map(decompressNode)
+    }
+  }
+
+  return {
+    type: 'root',
+    children: body.value.map(decompressNode)
+  }
+}

docs/modules/llms/runtime/server/tsconfig.json

@@ -0,0 +1,3 @@
+{
+	"extends": "../../../../.nuxt/tsconfig.server.json"
+}