Lunr integration for Astro; client-side search for statically hosted pages.
| 0 | import { createFilter } from '@rollup/pluginutils' | ||
| 1 | import fs from 'node:fs'; | ||
| 2 | + | import path from 'node:path'; | |
| 3 | import {rehype} from 'rehype' | ||
| 4 | import lunr from 'lunr'; | ||
| ... | |||
| 43 | } | ||
| 44 | |||
| 45 | - | function indexLunrElements(canonicalUrl, addToBulk){ | |
| 46 | + | function indexLunrDocuments(canonicalUrl, addToBulk){ | |
| 47 | return () => traverseReplace.bind(null, (node) => { | ||
| 48 | if(node.type === "element" && node.tagName === "lunr-document"){ | ||
| 49 | let doc = parseLunrDocument(node); | ||
| 50 | doc["canonicalUrl"] = canonicalUrl; | ||
| 51 | + | doc["__index__"] = node.properties.index; | |
| 52 | addToBulk(doc); | ||
| 53 | return [] | ||
| ... | |||
| 57 | |||
| 58 | |||
| 59 | - | function getViteConfiguration(options = {}) { | |
| 60 | - | var filter = createFilter(["**/pages/**/*.astro"], options.exclude, {}); | |
| 61 | - | return { | |
| 62 | - | plugins: [ | |
| 63 | - | { | |
| 64 | - | enforce: 'pre', // run transforms before other plugins can | |
| 65 | - | name: "lunr-rollup-plugin", | |
| 66 | - | // buildEnd() { | |
| 67 | - | // console.log("build end", this.getModuleIds); | |
| 68 | - | // // do something with this list | |
| 69 | - | // }, | |
| 70 | - | // async resolveId(id, importer, options) { | |
| 71 | - | // if(id === "/tree/master/src") console.log("res", id, importer, options) | |
| 72 | - | // return undefined; | |
| 73 | - | // }, | |
| 74 | - | transform(code, id) { | |
| 75 | - | if(!filter(id)) return null; | |
| 76 | - | const ast = this.parse(code); | |
| 77 | - | const ext = ast.body.filter(node => node.type === "ExportNamedDeclaration"); | |
| 78 | - | const indexDocumentAst = ext.find(node => node.declaration.declarations.find(n => n.id === "indexDocument")) | |
| 79 | - | if(!indexDocumentAst) return null; | |
| 80 | - | console.log("transform", id, indexDocumentAst); | |
| 81 | - | // const art = this.parse(code); | |
| 82 | - | // const source = await fs.promises.readFile(id, 'utf8').catch(err => console.log(err)); | |
| 83 | - | // console.log("loaded", id, source); | |
| 84 | - | } | |
| 85 | - | } | |
| 86 | - | ], | |
| 87 | - | }; | |
| 88 | - | } | |
| 89 | + | // function getViteConfiguration(options = {}) { | |
| 90 | + | // var filter = createFilter(["**/pages/**/*.astro"], options.exclude, {}); | |
| 91 | + | // return { | |
| 92 | + | // plugins: [ | |
| 93 | + | // { | |
| 94 | + | // enforce: 'pre', // run transforms before other plugins can | |
| 95 | + | // name: "lunr-rollup-plugin", | |
| 96 | + | // transform(code, id) { | |
| 97 | + | // if(!filter(id)) return null; | |
| 98 | + | // const ast = this.parse(code); | |
| 99 | + | // const ext = ast.body.filter(node => node.type === "ExportNamedDeclaration"); | |
| 100 | + | // const indexDocumentAst = ext.find(node => node.declaration.declarations.find(n => n.id === "indexDocument")) | |
| 101 | + | // if(!indexDocumentAst) return null; | |
| 102 | + | // console.log("transform", id, indexDocumentAst); | |
| 103 | + | // // const art = this.parse(code); | |
| 104 | + | // // const source = await fs.promises.readFile(id, 'utf8').catch(err => console.log(err)); | |
| 105 | + | // // console.log("loaded", id, source); | |
| 106 | + | // } | |
| 107 | + | // } | |
| 108 | + | // ], | |
| 109 | + | // }; | |
| 110 | + | // } | |
| 111 | |||
| 112 | |||
| 113 | - | export default function createPlugin({pathFilter, documentFilter}){ | |
| 114 | + | export default function createPlugin({pathFilter, subDir, documentFilter, initialize, mapDocument, verbose}){ | |
| 115 | let config = {}; | ||
| 116 | let pathsToIndex = [] | ||
| 117 | - | console.log("create plugin for lunr") | |
| 118 | return { | ||
| 119 | name: 'lunr-filenames', | ||
| ... | |||
| 123 | options.addRenderer({ | ||
| 124 | name: 'lunr-renderer', | ||
| 125 | - | serverEntrypoint: '@integrations/astro-lunr/renderer.js', | |
| 126 | + | serverEntrypoint: '@integrations/astro-lunr/server/renderer.js', | |
| 127 | }); | ||
| 128 | } | ||
| 129 | }, | ||
| 130 | - | 'astro:config:done': (options) => { | |
| 131 | - | console.log("config:done") | |
| 132 | - | }, | |
| 133 | - | 'astro:server:start': (options) => { | |
| 134 | - | console.log("astro:server:start", options); | |
| 135 | - | }, | |
| 136 | - | 'astro:build:start': (options) => { | |
| 137 | - | console.log("astro:build:start", options); | |
| 138 | - | }, | |
| 139 | - | 'astro:build:done': async ({pages, routes, dir}) => { | |
| 140 | - | console.log("build:done", pages[0], routes, JSON.stringify(routes[0].segments)); | |
| 141 | + | 'astro:build:done': async ({pages, dir}) => { | |
| 142 | + | let indexMap = new Map(); | |
| 143 | + | const addToIndexMap = (doc) => { | |
| 144 | + | if(documentFilter && !documentFilter(doc)){ | |
| 145 | + | return; | |
| 146 | + | } | |
| 147 | + | const {__index__: index, ...rest} = doc; | |
| 148 | + | if(!indexMap.has(index)){ | |
| 149 | + | indexMap.set(index, []); | |
| 150 | + | } | |
| 151 | + | indexMap.get(index).push({ | |
| 152 | + | ...rest, | |
| 153 | + | id: doc["id"] || crypto.createHash('md5').update(doc["canonicalUrl"]).digest('hex') | |
| 154 | + | }); | |
| 155 | + | } | |
| 156 | let documents = []; | ||
| 157 | for(let {pathname} of pages) { | ||
| ... | |||
| 161 | let url = new URL((pathname ? pathname + "/" : "") + "index.html", dir); | ||
| 162 | let content = fs.readFileSync(url, "utf-8"); | ||
| 163 | - | console.log(pathname, content.length); | |
| 164 | let newDocuments = []; | ||
| 165 | let hyped = await rehype() | ||
| 166 | - | .use(indexLunrElements(pathname, (doc) => newDocuments.push(doc))) | |
| 167 | + | .use(indexLunrDocuments(pathname, (doc) => newDocuments.push(doc))) | |
| 168 | .process(content); | ||
| 169 | if(newDocuments.length > 0) { | ||
| 170 | + | if(verbose){ | |
| 171 | + | console.log(`Indexing ${pathname}, found ${newDocuments.length} documents to index`); | |
| 172 | + | } | |
| 173 | fs.writeFileSync(url, String(hyped)); | ||
| 174 | - | documents.push(...newDocuments); | |
| 175 | + | newDocuments.forEach(addToIndexMap) | |
| 176 | } | ||
| 177 | } | ||
| 178 | - | documents = documents.map(doc => ({...doc, id: crypto.createHash('md5').update(doc["canonicalUrl"]).digest('hex')})) | |
| 179 | - | ||
| 180 | - | if(documentFilter){ | |
| 181 | - | documents = documents.filter(documentFilter); | |
| 182 | + | for(let [index, documents] of indexMap){ | |
| 183 | + | const idx = lunr(function () { | |
| 184 | + | initialize(this, lunr); | |
| 185 | + | documents.forEach(doc => this.add(doc)); | |
| 186 | + | }) | |
| 187 | + | if(mapDocument){ | |
| 188 | + | documents = documents.map(mapDocument); | |
| 189 | + | } | |
| 190 | + | fs.writeFileSync(new URL(path.join(subDir || "", index || "", 'idx.json'), dir), JSON.stringify(idx)); | |
| 191 | + | fs.writeFileSync(new URL(path.join(subDir || "", index || "", 'docs.json'), dir), JSON.stringify(documents)); | |
| 192 | } | ||
| 193 | - | ||
| 194 | - | lunr.tokenizer.separator = /[^\w]+/ | |
| 195 | - | const idx = lunr(function () { | |
| 196 | - | this.use(builder => { | |
| 197 | - | builder.pipeline.reset(); | |
| 198 | - | builder.searchPipeline.reset(); | |
| 199 | - | }) | |
| 200 | - | this.field("canonicalUrl", {boost: 0.01}); | |
| 201 | - | this.field("ref", {boost: 0.01}); | |
| 202 | - | this.field("oid", {boost: 0.01}); | |
| 203 | - | this.field("path", {boost: 0.1}); | |
| 204 | - | this.field("name", {boost: 10}); | |
| 205 | - | this.field("content"); | |
| 206 | - | this.metadataWhitelist = ['position']; | |
| 207 | - | documents.forEach(doc => this.add(doc)); | |
| 208 | - | }) | |
| 209 | - | const simplifiedDocuments = documents.map(doc => { | |
| 210 | - | let {content, summary, ...simple} = doc; | |
| 211 | - | return doc; | |
| 212 | - | }) | |
| 213 | - | fs.writeFileSync(new URL('lunr-index.json', dir), JSON.stringify(idx)); | |
| 214 | - | fs.writeFileSync(new URL('lunr-docs.json', dir), JSON.stringify(simplifiedDocuments)); | |
| 215 | } | ||
| 216 | } | ||
| 217 | } |