Lunr integration for Astro; client-side search for statically hosted pages.
| 0 | { | ||
| 1 | "name": "astro-lunr", | ||
| 2 | "version": "0.0.1", | ||
| 3 | "private": true, | ||
| 4 | + | "license" : "GPL-3.0", | |
| 5 | "description": "Lunr integration for Astro", | ||
| 6 | "type": "module", | ||
| 7 | + | "exports": { | |
| 8 | + | "./plugin.mjs": "./plugin.mjs", | |
| 9 | + | "./client/lunr.js": "./client/lunr.js", | |
| 10 | + | "./server/renderer.js": "./server/renderer.js" | |
| 11 | + | }, | |
| 12 | "dependencies": { | ||
| 13 | - | "lunr": "^2.3.9" | |
| 14 | - | }, | |
| 15 | - | "devDependencies": { | |
| 16 | + | "lunr": "^2.3.9", | |
| 17 | "rehype": "^12.0.1" | ||
| 18 | } |
| 0 | - | import { createFilter } from '@rollup/pluginutils' | |
| 1 | import fs from 'node:fs'; | ||
| 2 | import path from 'node:path'; | ||
| ... | |||
| 55 | } | ||
| 56 | |||
| 57 | - | ||
| 58 | - | // function getViteConfiguration(options = {}) { | |
| 59 | - | // var filter = createFilter(["**/pages/**/*.astro"], options.exclude, {}); | |
| 60 | - | // return { | |
| 61 | - | // plugins: [ | |
| 62 | - | // { | |
| 63 | - | // enforce: 'pre', // run transforms before other plugins can | |
| 64 | - | // name: "lunr-rollup-plugin", | |
| 65 | - | // transform(code, id) { | |
| 66 | - | // if(!filter(id)) return null; | |
| 67 | - | // const ast = this.parse(code); | |
| 68 | - | // const ext = ast.body.filter(node => node.type === "ExportNamedDeclaration"); | |
| 69 | - | // const indexDocumentAst = ext.find(node => node.declaration.declarations.find(n => n.id === "indexDocument")) | |
| 70 | - | // if(!indexDocumentAst) return null; | |
| 71 | - | // console.log("transform", id, indexDocumentAst); | |
| 72 | - | // // const art = this.parse(code); | |
| 73 | - | // // const source = await fs.promises.readFile(id, 'utf8').catch(err => console.log(err)); | |
| 74 | - | // // console.log("loaded", id, source); | |
| 75 | - | // } | |
| 76 | - | // } | |
| 77 | - | // ], | |
| 78 | - | // }; | |
| 79 | - | // } | |
| 80 | - | ||
| 81 | - | ||
| 82 | export default function createPlugin({pathFilter, subDir, documentFilter, initialize, mapDocument, verbose}){ | ||
| 83 | let config = {}; | ||
| ... | |||
| 90 | options.addRenderer({ | ||
| 91 | name: 'lunr-renderer', | ||
| 92 | - | serverEntrypoint: '@integrations/astro-lunr/server/renderer.js', | |
| 93 | + | serverEntrypoint: 'astro-lunr/server/renderer.js', | |
| 94 | }); | ||
| 95 | } | ||
| ... | |||
| 123 | if(newDocuments.length > 0) { | ||
| 124 | if(verbose){ | ||
| 125 | - | console.log(`Indexing ${pathname}, found ${newDocuments.length} documents to index`); | |
| 126 | + | console.log(`Indexing ${newDocuments.length} doc(s) from ${pathname}`); | |
| 127 | } | ||
| 128 | fs.writeFileSync(url, String(hyped)); | ||
| ... | |||
| 138 | documents = documents.map(mapDocument); | ||
| 139 | } | ||
| 140 | + | fs.mkdirSync(new URL(path.join(subDir || "", index || ""), dir), { recursive: true }); | |
| 141 | fs.writeFileSync(new URL(path.join(subDir || "", index || "", 'idx.json'), dir), JSON.stringify(idx)); | ||
| 142 | fs.writeFileSync(new URL(path.join(subDir || "", index || "", 'docs.json'), dir), JSON.stringify(documents)); | ||
| ... | |||
| 145 | } | ||
| 146 | } |
| 0 | |||
| 1 | |||
| 2 | # astro-lunr | ||
| 3 | |||
| 4 | - | [Lunr](https://lunrjs.com) integration for [Astro](https://astro.build/) | |
| 5 | + | [Lunr](https://lunrjs.com) integration for [Astro](https://astro.build/). | |
| 6 | |||
| 7 | + | See example usage in [astro-git-view](https://github.com/siverv/astro-git-view) | |
| 8 | + | ||
| 9 | + | ## Usage | |
| 10 | + | ||
| 11 | + | ```js | |
| 12 | + | import { defineConfig } from 'astro/config'; | |
| 13 | + | import astroLunr from 'astro-lunr/plugin.mjs'; | |
| 14 | + | export default defineConfig({ | |
| 15 | + | integrations: [ | |
| 16 | + | astroLunr({}) | |
| 17 | + | ] | |
| 18 | + | }); | |
| 19 | + | ``` | |
| 20 | + | ||
| 21 | ## Config | ||
| 22 | |||
| 23 | + | pathFilter, subDir, documentFilter, initialize, mapDocument, verbose | |
| 24 | + | ||
| 25 | + | | Config field | Type | Value | | |
| 26 | + | |:---------------- |:------------------ |:-------------------------------------------- | | |
| 27 | + | | `subDir` | string | Subdirectory to store the created idx.json and docs.json files. | | |
| 28 | + | | `pathFilter` | (string) => boolean | Filter for paths that should be searched for `<lunr-document/>`-elements | | |
| 29 | + | | `documentFilter` | (string) => boolean | Filter for documents should be included in the final index, after they are found by searching the pre-generated pages | | |
| 30 | + | | `initialize` | (lunr.Builder, lunr) => void | Lunr-specific setup. E.g. fields to index and pipeline-adjustments | | |
| 31 | + | | `mapDocument` | (Object) => Object | Transform the documents before storing them in docs.json | | |
| 32 | + | | `verbose` | boolean | Debug log | | |
| 33 | + | ||
| 34 | + | ||
| 35 | + | ### Example from astro-git-view | |
| 36 | + | ||
| 37 | + | ```js | |
| 38 | + | astroLunr({ | |
| 39 | + | subDir: "lunr", | |
| 40 | + | pathFilter: (pathname) => { | |
| 41 | + | return pathname.match(/\w+\/tree\//); | |
| 42 | + | }, | |
| 43 | + | documentFilter: (doc) => { | |
| 44 | + | return doc.ref === "master" && !doc.canonicalUrl.includes("package-lock.json"); | |
| 45 | + | }, | |
| 46 | + | initialize: (builder, lunr) => { | |
| 47 | + | lunr.tokenizer.separator = /[^\w]+/; | |
| 48 | + | builder.pipeline.reset(); | |
| 49 | + | builder.searchPipeline.reset(); | |
| 50 | + | builder.field("ref", {boost: 0.01}); | |
| 51 | + | builder.field("oid", {boost: 0.01}); | |
| 52 | + | builder.field("path", {boost: 0.1}); | |
| 53 | + | builder.field("name", {boost: 10}); | |
| 54 | + | builder.field("content"); | |
| 55 | + | builder.metadataWhitelist = ["position"]; | |
| 56 | + | }, | |
| 57 | + | mapDocument: (doc) => doc, | |
| 58 | + | verbose: false | |
| 59 | + | }) | |
| 60 | + | ``` | |
| 61 | + | ||
| 62 | ## Indexing documents | ||
| 63 | |||
| 64 | + | Indexing is done automatically during build-time, by searching the genereated pages for the `<lunr-document/>`-element on any of the generated pages. These elements are removed from the final version of the build-output, and therefore only affects the two generated index-files `idx.json` and `docs.json`. | |
| 65 | + | ||
| 66 | + | Multiple indexes are supported by supplying the index-attribute to the `<lunr-document/>`-elements. Each index will create their own `idx.json`/`docs.json` pair. | |
| 67 | + | ||
| 68 | + | ### Example from astro-git-view | |
| 69 | + | ||
| 70 | + | ```jsx | |
| 71 | + | <lunr-document index={repo.getName()}> | |
| 72 | + | <lunr-field name="repo" value={repo.getName()}/> | |
| 73 | + | <lunr-field name="ref" value={ref}/> | |
| 74 | + | <lunr-field name="path" value={path}/> | |
| 75 | + | <lunr-field name="base" value={path.split("/").slice(0,-1).join("/")}/> | |
| 76 | + | <lunr-field name="name" value={name}/> | |
| 77 | + | <lunr-field name="oid" value={oid}/> | |
| 78 | + | <lunr-field name="type" value={"blob"}/> | |
| 79 | + | <lunr-field name="extension" value={name.split(".").pop()}/> | |
| 80 | + | <lunr-text name="content">{content}</lunr-text> | |
| 81 | + | </lunr-document> | |
| 82 | + | ``` | |
| 83 | + | ||
| 84 | ## Searching | ||
| 85 | |||
| 86 | + | `astro-lunr` gives two functions to search `search` and `enrich`. `async search(query, index)` loads and deserializes the `idx.json` file, performs the search, and returns the hits from lunr. `async enrich(hits, index)` loads the `docs.json` file and enriches the result from lunr with the documents that were matches. | |
| 87 | + | ||
| 88 | + | As the two json-files often can be megabytes in size, it is recommended to not | |
| 89 | + | ||
| 90 | + | ### Example | |
| 91 | + | ||
| 92 | + | ```js | |
| 93 | + | import {search, enrich} from 'astro-lunr/client/lunr.js'; | |
| 94 | + | ||
| 95 | + | search("query", "index") | |
| 96 | + | .then(enrich) | |
| 97 | + | .then((result) => result.forEach( | |
| 98 | + | ({hit, doc}) => console.log(hit, doc))) | |
| 99 | + | ``` | |
| 100 | + | ||
| 101 | + | ### Searching in Dev-mode | |
| 102 | + | ||
| 103 | + | Due to the nature of indexing, to properly search in dev-mode, one needs to first build the page at least once to create the index | |
| 104 | + | ||
| 105 | + | Furthermore, to make the dev-server accept the index-files, you might need to symbolically link the index-files from the public-folder. These should be included in `.gitignore` or equivalent. | |
| 106 | + | ||
| 107 | + | Example of how this is done automatically for `astro-git-view`: | |
| 108 | + | ||
| 109 | + | ```json | |
| 110 | + | "scripts": { | |
| 111 | + | "dev": "(ln -s ../dist/lunr ./public/lunr 2> /dev/null || true) && astro dev", | |
| 112 | + | "start": "(ln -s ../dist/lunr ./public/lunr 2> /dev/null || true) && astro dev", | |
| 113 | + | "build": "(rm ./public/lunr 2> /dev/null || true) && astro build", | |
| 114 | + | "preview": "astro preview" | |
| 115 | + | }, | |
| 116 | + | ``` | |
| 117 | + | ||
| 118 | + | A potential future solution would be to "trick" the dev-server into serving certain files from `./dist` without the end-user needing to think, or by writing the files to both dist and public. | |
| 119 | + | ||
| 120 | + | ### Searching in SSR-mode | |
| 121 | + | ||
| 122 | + | To properly search files that are not usually genereated in the build-step, you would need to have a separate build-step that includes all pages that might be generated. | |
| 123 | + |