From c5bb540519df5e78228e3eb3db705050ebc4fe53 Mon Sep 17 00:00:00 2001 From: Farnabaz Date: Wed, 5 Feb 2025 18:25:39 +0100 Subject: [PATCH] docs(llms): generate `llms.txt` from content (#3246) --- docs/modules/llms/module.ts | 47 +++++++++++ .../runtime/server/routes/llms.txt.get.ts | 56 +++++++++++++ .../server/routes/llms_full.txt.get.ts | 78 +++++++++++++++++++ .../modules/llms/runtime/server/tsconfig.json | 3 + docs/nuxt.config.ts | 47 ++++++++++- 5 files changed, 230 insertions(+), 1 deletion(-) create mode 100644 docs/modules/llms/module.ts create mode 100644 docs/modules/llms/runtime/server/routes/llms.txt.get.ts create mode 100644 docs/modules/llms/runtime/server/routes/llms_full.txt.get.ts create mode 100644 docs/modules/llms/runtime/server/tsconfig.json diff --git a/docs/modules/llms/module.ts b/docs/modules/llms/module.ts new file mode 100644 index 00000000..1a13deea --- /dev/null +++ b/docs/modules/llms/module.ts @@ -0,0 +1,47 @@ +import { addPrerenderRoutes, addServerScanDir, createResolver, defineNuxtModule, useLogger } from '@nuxt/kit' +import type { SQLOperator } from '@nuxt/content' + +export interface ModuleOptions { + domain: string + sections: Array<{ + title: string + collection: string + description?: string + filters?: Array<{ + field: string + operator: SQLOperator + value?: string + }> + }> + title?: string + description?: string + notes?: string[] +} + +export default defineNuxtModule({ + meta: { + name: 'llms', + configKey: 'llms' + }, + setup(options, nuxt) { + const { resolve } = createResolver(import.meta.url) + const logger = useLogger('llms') + + nuxt.options.runtimeConfig.llms = { + domain: options.domain, + title: options.title, + description: options.description, + notes: options.notes, + sections: options.sections || [{ title: 'Docs', collection: 'content' }] + } + + if (!options.domain) { + logger.warn('Please provide a domain for the LLMs module. LLMS docs require a domain to be set.') + } + + addServerScanDir(resolve('runtime/server')) + + addPrerenderRoutes('/llms.txt') + addPrerenderRoutes('/llms_full.txt') + } +}) diff --git a/docs/modules/llms/runtime/server/routes/llms.txt.get.ts b/docs/modules/llms/runtime/server/routes/llms.txt.get.ts new file mode 100644 index 00000000..f68f276c --- /dev/null +++ b/docs/modules/llms/runtime/server/routes/llms.txt.get.ts @@ -0,0 +1,56 @@ +import { joinURL } from 'ufo' +import type { ModuleOptions } from '~~/modules/llms/module' + +export default eventHandler(async (event) => { + const options = useRuntimeConfig(event).llms as ModuleOptions + + const llms = [ + `# ${options.title || 'Documentation'}` + ] + + if (options.description) { + llms.push(`> ${options.description}`) + } + + llms.push( + '## Documentation Sets', + `- [Complete Documentation](${joinURL(options.domain, '/llms_full.txt')}): The complete documentation including all content` + ) + + for (const section of options.sections) { + // @ts-expect-error - typecheck does not derect server querryCollection + const query = queryCollection(event, section.collection) + .select('path', 'title', 'description') + .where('path', 'NOT LIKE', '%/.navigation') + + if (section.filters) { + for (const filter of section.filters) { + query.where(filter.field, filter.operator, filter.value) + } + } + + const docs = await query.all() + + const links = docs.map((doc) => { + return `- [${doc.title}](${joinURL(options.domain, doc.path)}): ${doc.description}` + }) + + llms.push(`## ${section.title}`) + + if (section.description) { + llms.push(section.description) + } + + llms.push(links.join('\n')) + } + + if (options.notes && options.notes.length) { + llms.push( + '## Notes', + (options.notes || []).map(note => `- ${note}`).join('\n') + ) + } + + setHeader(event, 'Content-Type', 'text/plain') + return llms.join('\n\n') +}) diff --git a/docs/modules/llms/runtime/server/routes/llms_full.txt.get.ts b/docs/modules/llms/runtime/server/routes/llms_full.txt.get.ts new file mode 100644 index 00000000..69d0fbdf --- /dev/null +++ b/docs/modules/llms/runtime/server/routes/llms_full.txt.get.ts @@ -0,0 +1,78 @@ +import { joinURL, hasProtocol } from 'ufo' +import type { ModuleOptions } from '~~/modules/llms/module' +import { stringifyMarkdown } from '@nuxtjs/mdc/runtime' +import type { MDCRoot } from '@nuxtjs/mdc' + +export default eventHandler(async (event) => { + const options = useRuntimeConfig(event).llms as ModuleOptions + + const llms = [] + + for (const section of options.sections) { + // @ts-expect-error - typecheck does not derect server querryCollection + const query = queryCollection(event, section.collection) + .where('path', 'NOT LIKE', '%/.navigation') + + if (section.filters) { + for (const filter of section.filters) { + query.where(filter.field, filter.operator, filter.value) + } + } + + const docs = await query.all() + + for (const doc of docs) { + let markdown = await stringifyMarkdown(decompressBody(doc.body, options), {}) + + if (!markdown?.trim().startsWith('# ')) { + markdown = `# ${doc.title}\n\n${markdown}` + } + llms.push(markdown) + } + } + + if (options.notes && options.notes.length) { + llms.push( + '## Notes', + (options.notes || []).map(note => `- ${note}`).join('\n') + ) + } + + setHeader(event, 'Content-Type', 'text/plain') + return llms.join('\n\n') +}) + +// decompress utils is part of Content module and not exposed yet +// We can refactor this after exposing the utils +function decompressBody(body: any, options: ModuleOptions): MDCRoot { + const linkProps = ['href', 'src', 'to'] + + function decompressNode(input: any) { + if (typeof input === 'string') { + return { + type: 'text', + value: input + } + } + + const [tag, props, ...children] = input + + for (const prop of linkProps) { + if (props[prop] && !hasProtocol(props[prop])) { + props[prop] = joinURL(options.domain, props[prop]) + } + } + + return { + type: 'element', + tag, + props, + children: children.map(decompressNode) + } + } + + return { + type: 'root', + children: body.value.map(decompressNode) + } +} diff --git a/docs/modules/llms/runtime/server/tsconfig.json b/docs/modules/llms/runtime/server/tsconfig.json new file mode 100644 index 00000000..cba8b805 --- /dev/null +++ b/docs/modules/llms/runtime/server/tsconfig.json @@ -0,0 +1,3 @@ +{ + "extends": "../../../../.nuxt/tsconfig.server.json" +} \ No newline at end of file diff --git a/docs/nuxt.config.ts b/docs/nuxt.config.ts index ad106fc5..fd91f763 100644 --- a/docs/nuxt.config.ts +++ b/docs/nuxt.config.ts @@ -22,7 +22,8 @@ export default defineNuxtConfig({ nuxt.hook('components:dirs', (dirs) => { dirs.unshift({ path: resolve('./app/components/content/examples'), pathPrefix: false, prefix: '', global: true }) }) - } + }, + '~~/modules/llms/module' ], app: { @@ -147,6 +148,50 @@ export default defineNuxtConfig({ image: { provider: 'ipx' }, + llms: { + domain: 'https://ui3.nuxt.dev', + title: 'Nuxt UI v3', + description: 'A comprehensive, Nuxt-integrated UI library providing a rich set of fully-styled, accessible and highly customizable components for building modern web applications.', + sections: [ + { + title: 'Getting Started', + collection: 'content', + filters: [ + { + field: 'path', + operator: 'LIKE', + value: '/getting-started%' + } + ] + }, + { + title: 'Components', + collection: 'content', + filters: [ + { + field: 'path', + operator: 'LIKE', + value: '/components/%' + } + ] + }, + { + title: 'Composables', + collection: 'content', + filters: [ + { + field: 'path', + operator: 'LIKE', + value: '/composables/%' + } + ] + } + ], + notes: [ + 'The documentation excludes Nuxt UI v2 content.', + 'The content is automatically generated from the same source as the official documentation.' + ] + }, uiPro: { license: 'oss'