diff --git a/README.md b/README.md index e6210b3f..e76a7985 100644 --- a/README.md +++ b/README.md @@ -24,3 +24,23 @@ For that, all you need to do is follow the instructions below: 3. Finally to bypass CORS you will want to get this Chrome [extension](https://chrome.google.com/webstore/detail/moesif-origin-cors-change/digfbfaphojjndkpccljibejjbppifbc) When you are done, just deploy your `openapi.yml` as you normally would. And don't forget to update your client sdk [here](https://github.com/epilot-dev/sdk-js). + +## Search + +The site search is a local plugin in [`src/plugins/search`](src/plugins/search): + +- `indexer.js` runs after `npm run build` and writes `build/search-index.json`. It indexes every section of the rendered docs and changelog pages, and every operation of the OpenAPI specs listed in `redoc.config.js`. +- `engine.js` ranks results in the browser (BM25F with field weights, stemming, prefix and typo matching, title and phrase boosts). +- `theme/SearchBar` is the search dialog (`Ctrl/⌘ K` or `/`). + +The dev server (`npm start`) has no index of its own: it serves the one from the last `npm run build`. + +After changing the ranking, verify it against the built index: + +```bash +npm run build +npm run test:search # relevance checks +npm run test:search -- "webhook signature" # print the results of a query +``` + +To keep an element out of the index, add a `data-search-ignore` attribute to it. diff --git a/docusaurus.config.js b/docusaurus.config.js index 0205563e..e6cb5e6a 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -94,7 +94,7 @@ const DOCS_URL = process.env.DOCS_URL || 'https://docs.epilot.io' }, } }, - require.resolve('@cmfcmf/docusaurus-search-local'), + [require.resolve('./src/plugins/search/index.js'), { specs }], [ '@docusaurus/plugin-client-redirects', { diff --git a/i18n/en/code.json b/i18n/en/code.json deleted file mode 100644 index 10baf53d..00000000 --- a/i18n/en/code.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cmfcmf/d-s-l.searchBar.placeholder": { - "message": "Search docs...", - "description": "Placeholder shown in the searchbar" - } -} diff --git a/package-lock.json b/package-lock.json index f47f90bd..3dc3fea2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,6 @@ "name": "epilot-dev-handbook", "version": "0.0.0", "dependencies": { - "@cmfcmf/docusaurus-search-local": "^0.10.0", "@docusaurus/core": "2.0.0-beta.15", "@docusaurus/plugin-client-redirects": "^2.0.0-beta.15", "@docusaurus/plugin-content-docs": "2.0.0-beta.15", @@ -22,8 +21,10 @@ "@stoplight/markdown-viewer": "^5.7.1", "@stoplight/mosaic": "^1.53.5", "@svgr/webpack": "^5.5.0", + "cheerio": "1.0.0-rc.12", "clsx": "^1.1.1", "file-loader": "^6.2.0", + "fs-extra": "^10.1.0", "lottie-web": "^5.13.0", "mdx-mermaid": "^1.3.2", "mermaid": "^11.12.2", @@ -31,7 +32,8 @@ "react": "^17.0.1", "react-dom": "^17.0.1", "redocusaurus": "^0.6.0", - "url-loader": "^4.1.1" + "url-loader": "^4.1.1", + "yaml": "^1.10.3" }, "devDependencies": { "@docusaurus/module-type-aliases": "2.0.0-beta.15", @@ -55,68 +57,6 @@ "node": ">=0.10.0" } }, - "node_modules/@algolia/autocomplete-core": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/@algolia/autocomplete-core/-/autocomplete-core-1.13.0.tgz", - "integrity": "sha512-0v3mHfkvJBVx0aO1U290EHaLPp9pkUL8zkgbVY0JlitItrbXfYYHQHtNs1TxpA63mQAD0K0LyLzO2x+uWiBbGQ==", - "dependencies": { - "@algolia/autocomplete-plugin-algolia-insights": "1.13.0", - "@algolia/autocomplete-shared": "1.13.0" - } - }, - "node_modules/@algolia/autocomplete-js": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/@algolia/autocomplete-js/-/autocomplete-js-1.13.0.tgz", - "integrity": "sha512-gw2jbkIzSH+xljX3yoOg+5nfJwMh7jqw5T/jy/WPwgmPhn5Mv6PmosCM0huGwH2E88nwxNlY2AhbkDrS4qceAw==", - "dependencies": { - "@algolia/autocomplete-core": "1.13.0", - "@algolia/autocomplete-preset-algolia": "1.13.0", - "@algolia/autocomplete-shared": "1.13.0", - "htm": "^3.1.1", - "preact": "^10.13.2" - }, - "peerDependencies": { - "@algolia/client-search": ">= 4.5.1 < 6", - "algoliasearch": ">= 4.9.1 < 6" - } - }, - "node_modules/@algolia/autocomplete-plugin-algolia-insights": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/@algolia/autocomplete-plugin-algolia-insights/-/autocomplete-plugin-algolia-insights-1.13.0.tgz", - "integrity": "sha512-Q0rRUZ72x7piqvJKi1//SBZvoImnYdJLRC7Yaa0rwKtkIVQFl6MmZw/p4AEDSWIu5HY3Ki3bzgYxeDyhm//P/w==", - "dependencies": { - "@algolia/autocomplete-shared": "1.13.0" - }, - "peerDependencies": { - "search-insights": ">= 1 < 3" - } - }, - "node_modules/@algolia/autocomplete-preset-algolia": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/@algolia/autocomplete-preset-algolia/-/autocomplete-preset-algolia-1.13.0.tgz", - "integrity": "sha512-IlanOCLT2EvfygX5cGFR5iKgfhQB0MqCv163ldctq8l0QCVdEOM1VLIQhl0tB3ViJc5XKUB8QZ7V+DcSVtZAuQ==", - "dependencies": { - "@algolia/autocomplete-shared": "1.13.0" - }, - "peerDependencies": { - "@algolia/client-search": ">= 4.9.1 < 6", - "algoliasearch": ">= 4.9.1 < 6" - } - }, - "node_modules/@algolia/autocomplete-shared": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/@algolia/autocomplete-shared/-/autocomplete-shared-1.13.0.tgz", - "integrity": "sha512-YB7JlPl1coHai3Xd4OdNIMavAMbgx8eHPH9nlEgcrCqCx57njh0qReruTMRxaThBaWIkkl47jZlUnKvb8MjGGQ==", - "peerDependencies": { - "@algolia/client-search": ">= 4.9.1 < 6", - "algoliasearch": ">= 4.9.1 < 6" - } - }, - "node_modules/@algolia/autocomplete-theme-classic": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/@algolia/autocomplete-theme-classic/-/autocomplete-theme-classic-1.13.0.tgz", - "integrity": "sha512-YAyfcpi+VJ0h5PUTThDmc/V2OB47RNlvIBQgffzrjAw5vDkoBcAj5bsReJW8/QtLnRGB85XhrmWoYFtP4W3HgQ==" - }, "node_modules/@algolia/cache-browser-local-storage": { "version": "4.22.1", "resolved": "https://registry.npmjs.org/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.22.1.tgz", @@ -2216,30 +2156,6 @@ "integrity": "sha512-YslZMgtJUyuMbZ+aKvfF3x1f5liK4mWNxghFRv7jqRR9C3R3fAOGTTKvxXDa2Y1s9zSbcpuO0cAxDYsc9SrXoQ==", "license": "Apache-2.0" }, - "node_modules/@cmfcmf/docusaurus-search-local": { - "version": "0.10.0", - "resolved": "https://registry.npmjs.org/@cmfcmf/docusaurus-search-local/-/docusaurus-search-local-0.10.0.tgz", - "integrity": "sha512-X6xabJvvbbgrqgkYUUHSER5kswonKRZatNDJmZTsNiCxFKCephbDL1Kui6QLN6jmorPNWSupj7aMSP3HObHnUg==", - "dependencies": { - "@algolia/autocomplete-js": "^1.5.1", - "@algolia/autocomplete-theme-classic": "^1.5.1", - "@algolia/client-search": "^4.12.0", - "algoliasearch": "^4.12.0", - "cheerio": "^1.0.0-rc.9", - "clsx": "^1.1.1", - "lunr-languages": "^1.4.0", - "mark.js": "^8.11.1" - }, - "peerDependencies": { - "@docusaurus/core": "^v2.0.0-beta.15", - "nodejieba": "^2.5.0" - }, - "peerDependenciesMeta": { - "nodejieba": { - "optional": true - } - } - }, "node_modules/@discoveryjs/json-ext": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz", @@ -8572,6 +8488,7 @@ "version": "1.0.0-rc.12", "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz", "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==", + "license": "MIT", "dependencies": { "cheerio-select": "^2.1.0", "dom-serializer": "^2.0.0", @@ -12177,6 +12094,7 @@ "version": "10.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", + "license": "MIT", "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -12996,11 +12914,6 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, - "node_modules/htm": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/htm/-/htm-3.1.1.tgz", - "integrity": "sha512-983Vyg8NwUE7JkZ6NmOqpCZ+sh1bKv2iYTlUkzlWmA5JD2acKoxd4KVxbMmxX/85mtfdnDmTFoNKcg5DGAvxNQ==" - }, "node_modules/html-encoding-sniffer": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz", @@ -14693,11 +14606,6 @@ "resolved": "https://registry.npmjs.org/lunr/-/lunr-2.3.9.tgz", "integrity": "sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow==" }, - "node_modules/lunr-languages": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/lunr-languages/-/lunr-languages-1.14.0.tgz", - "integrity": "sha512-hWUAb2KqM3L7J5bcrngszzISY4BxrXn/Xhbb9TTCJYEGqlR1nG67/M14sp09+PTIRklobrn57IAxcdcO/ZFyNA==" - }, "node_modules/magic-error": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/magic-error/-/magic-error-0.0.1.tgz", @@ -17276,15 +17184,6 @@ "postcss": "^8.2.15" } }, - "node_modules/preact": { - "version": "10.19.3", - "resolved": "https://registry.npmjs.org/preact/-/preact-10.19.3.tgz", - "integrity": "sha512-nHHTeFVBTHRGxJXKkKu5hT8C/YWBkPso4/Gad6xuj5dbptt9iF9NZr9pHbPhBrnT2klheu7mHTxTZ/LjwJiEiQ==", - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/preact" - } - }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -22609,9 +22508,10 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "node_modules/yaml": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz", - "integrity": "sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==", + "version": "1.10.3", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.3.tgz", + "integrity": "sha512-vIYeF1u3CjlhAFekPPAk2h/Kv4T3mAkMox5OymRiJQB0spDP10LHvt+K7G9Ny6NuuMAb25/6n1qyUjAcGNf/AA==", + "license": "ISC", "engines": { "node": ">= 6" } diff --git a/package.json b/package.json index 854cf30f..83490f54 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "write-translations": "docusaurus write-translations", "write-heading-ids": "docusaurus write-heading-ids", "typecheck": "tsc", + "test:search": "node scripts/check-search-relevance.js", "lint": "eslint src --ext js,jsx,ts,tsx,json", "lint:fix": "eslint src --ext js,jsx,ts,tsx,json --fix", "pdf": "./scripts/build-pdf.sh", @@ -27,7 +28,6 @@ "update-pricing-playground": "./scripts/update-pricing-playground.sh" }, "dependencies": { - "@cmfcmf/docusaurus-search-local": "^0.10.0", "@docusaurus/core": "2.0.0-beta.15", "@docusaurus/plugin-client-redirects": "^2.0.0-beta.15", "@docusaurus/plugin-content-docs": "2.0.0-beta.15", @@ -41,8 +41,10 @@ "@stoplight/markdown-viewer": "^5.7.1", "@stoplight/mosaic": "^1.53.5", "@svgr/webpack": "^5.5.0", + "cheerio": "1.0.0-rc.12", "clsx": "^1.1.1", "file-loader": "^6.2.0", + "fs-extra": "^10.1.0", "lottie-web": "^5.13.0", "mdx-mermaid": "^1.3.2", "mermaid": "^11.12.2", @@ -50,7 +52,8 @@ "react": "^17.0.1", "react-dom": "^17.0.1", "redocusaurus": "^0.6.0", - "url-loader": "^4.1.1" + "url-loader": "^4.1.1", + "yaml": "^1.10.3" }, "devDependencies": { "@docusaurus/module-type-aliases": "2.0.0-beta.15", diff --git a/scripts/check-search-relevance.js b/scripts/check-search-relevance.js new file mode 100644 index 00000000..400f28a7 --- /dev/null +++ b/scripts/check-search-relevance.js @@ -0,0 +1,155 @@ +/* eslint-disable @typescript-eslint/no-var-requires */ +/** + * Relevance checks for the site search. + * + * Runs real queries against the index of the last build and verifies that the + * page a reader is looking for is among the first results. Run it after + * changing the ranking in src/plugins/search/engine.js: + * + * npm run build && npm run test:search + * + * Pass queries as arguments to print their results instead: + * + * npm run test:search -- "webhook signature" getEntity + */ +const fs = require('fs'); +const path = require('path'); + +const { createIndex } = require('../src/plugins/search/engine'); + +const INDEX_PATH = path.join(__dirname, '..', 'build', 'search-index.json'); + +// [query, expected page (or one of several), worst acceptable position] +const CASES = [ + // Topics: the page about the topic comes first + ['webhooks', '/docs/integrations/webhooks/', 1], + ['authentication', '/docs/auth/authentication/', 1], + ['access tokens', '/docs/auth/access-tokens/', 1], + ['token types', '/docs/auth/token-types/', 1], + ['permissions', ['/docs/auth/permissions/', '/docs/apps/configure-permissions/'], 1], + ['grant actions', '/docs/auth/grant-actions/', 1], + ['sso', '/docs/sso/single-sign-on/', 1], + ['single sign on', '/docs/sso/single-sign-on/', 1], + ['mfa', '/docs/auth/mfa/', 1], + ['passkeys', '/docs/auth/passkeys/', 1], + ['pii anonymization', '/docs/auth/anonymization/', 1], + ['rate limits', '/docs/integrations/rate-limits/', 1], + ['blueprints', '/docs/blueprints/intro/', 1], + ['sandbox', '/docs/blueprints/sandboxes/', 3], + ['entity mapping', '/docs/automation/entity-mapping/', 1], + ['core entities', '/docs/entities/core-entities/', 1], + ['relations', '/docs/entities/relations/', 1], + ['taxonomy', '/docs/entities/taxonomies/', 1], + ['changesets', '/docs/entities/changesets/', 1], + ['customer portal', '/docs/portals/customer-portal/', 2], + ['pricing models', '/docs/pricing/pricing-models/', 1], + ['orders', '/docs/pricing/orders/', 2], + ['document generation', '/docs/files/document-generation/', 1], + ['template variables', '/docs/templates/template-variables/', 1], + ['email templates', '/docs/templates/email-templates/', 1], + ['custom css', '/docs/journeys/custom-css/', 1], + ['journey embed', ['/docs/journeys/sdk/', '/docs/journeys/embedding/'], 1], + ['web components', '/docs/journeys/web-components/', 1], + ['sdk', '/docs/sdk/overview/', 1], + ['cli', '/docs/cli/overview/', 1], + ['mcp', '/docs/agent-toolkit/mcp-server/', 1], + ['zapier', '/docs/integrations/zapier/', 1], + ['integration toolkit', '/docs/integrations/integration-toolkit/overview/', 1], + ['meter readings', '/docs/integrations/integration-toolkit/inbound/meter-readings/', 2], + ['static ip', '/docs/integrations/webhooks/static-ip/', 1], + ['audit logs', '/docs/audit-logs/', 1], + ['app manifest', '/docs/apps/app-manifest/', 1], + [ + 'custom journey block', + ['/docs/apps/components/custom-journey-block/', '/docs/apps/how-to-guides/build-a-custom-jb/'], + 1, + ], + ['datalake', '/docs/datalake/epilot-datalake/', 1], + ['workflows', '/docs/workflows/intro/', 1], + ['automation', '/docs/automation/intro/', 1], + ['environments secrets', '/docs/environments/environments-secrets/', 1], + ['volt ui', '/docs/architecture/design-system/', 1], + + // Questions and word forms + ['how to authenticate', '/docs/auth/authentication/', 3], + ['how do I verify a webhook signature', '/docs/integrations/webhooks/security/', 1], + ['installing blueprints', '/docs/blueprints/installing-a-blueprint/', 1], + ['embed a journey', ['/docs/journeys/sdk/', '/docs/journeys/embedding/'], 1], + ['generate documents', '/docs/files/document-generation/', 2], + ['webhook signature', '/docs/integrations/webhooks/security/', 1], + ['file upload', ['/docs/files/file-api/', '/api/file/'], 3], + + // Identifiers, endpoints and headers + ['getEntity', '/api/entity/', 1], + ['createEntity', '/api/entity/', 1], + ['searchEntities', '/api/entity/', 1], + ['create entity', '/api/entity/', 2], + ['triggerWebhook', '/api/webhooks/', 1], + ['POST /v1/entity/{slug}', '/api/entity/', 1], + ['/v1/webhooks/configs', '/api/webhooks/', 1], + ['x-epilot-org-id', '/docs/collaboration/sharing/', 3], + ['entity api', ['/docs/entities/entity-api/', '/api/entity/'], 1], + + // Typing in progress + ['authen', '/docs/auth/authentication/', 1], + ['webh', '/docs/integrations/webhooks/', 1], + ['blueprin', '/docs/blueprints/intro/', 1], + ['entity map', '/docs/automation/entity-mapping/', 1], + + // Typos + ['authentcation', '/docs/auth/authentication/', 1], + ['webhok', '/docs/integrations/webhooks/', 1], + ['entiy mapping', '/docs/automation/entity-mapping/', 1], + ['bluprint', '/docs/blueprints/intro/', 3], +]; + +if (!fs.existsSync(INDEX_PATH)) { + console.error(`No search index at ${INDEX_PATH}. Run "npm run build" first.`); + process.exit(1); +} + +const index = createIndex(JSON.parse(fs.readFileSync(INDEX_PATH, 'utf-8'))); + +index.build(); + +const plain = (segments) => segments.map(([text, isMatch]) => (isMatch ? `[${text}]` : text)).join(''); +const queries = process.argv.slice(2); + +if (queries.length) { + for (const query of queries) { + const result = index.search(query); + + console.log(`\n"${query}": ${result.total} pages`); + + for (const page of result.pages) { + console.log(` ${page.score.toFixed(1).padStart(6)} ${plain(page.title)} (${page.url})`); + + for (const section of page.sections) { + const label = section.heading ? plain(section.heading) : 'page'; + const text = section.excerpt ? plain(section.excerpt.text) : ''; + + console.log(` ${section.score.toFixed(1)} ${label}: ${text.slice(0, 110)}`); + } + } + } + process.exit(0); +} + +let failures = 0; + +for (const [query, expected, worst] of CASES) { + const accepted = [].concat(expected); + const urls = index.search(query, { limit: 10 }).pages.map((page) => page.url); + const position = urls.findIndex((url) => accepted.includes(url)) + 1; + + if (position === 0 || position > worst) { + failures++; + console.log( + `FAIL "${query}": expected ${accepted.join(' or ')} in top ${worst}, got position ${position || 'none'}`, + ); + urls.slice(0, 4).forEach((url, i) => console.log(` ${i + 1}. ${url}`)); + } +} + +console.log(`\n${CASES.length - failures}/${CASES.length} search relevance checks passed`); +process.exit(failures ? 1 : 0); diff --git a/src/css/custom.css b/src/css/custom.css index 0c38c78c..be7e0337 100644 --- a/src/css/custom.css +++ b/src/css/custom.css @@ -147,25 +147,10 @@ html[data-theme='dark'] .button.button--primary { gap: 0; } -.navbar__items--right .navbar__search, -.navbar__items--right .dsla-search-wrapper { +.navbar__items--right .navbar__search { position: absolute; left: 50%; transform: translateX(-50%); - min-width: 200px; -} - -.aa-DetachedSearchButton, -.dsla-search-wrapper .aa-Autocomplete .aa-Form, -.dsla-search-wrapper .dsla-search-field input, -.dsla-search-wrapper input[type="search"], -.navbar__search-input { - border-radius: 0.75rem !important; -} - -.aa-DetachedContainer--modal .aa-Form { - border-radius: 0.75rem !important; - margin: 8px; } /* Navbar button links */ @@ -216,10 +201,6 @@ html[data-theme='dark'] .button.button--primary { } -.aa-DetachedSearchButtonPlaceholder { - padding-right: 10px; -} - /* Hide navbar buttons and fix search on mobile */ @media (max-width: 996px) { .navbar-button, @@ -227,25 +208,12 @@ html[data-theme='dark'] .button.button--primary { display: none; } - .navbar__items--right .navbar__search, - .navbar__items--right .dsla-search-wrapper { + .navbar__items--right .navbar__search { position: static; transform: none; - min-width: 0; } } -/* Fix search bar colors in dark mode */ -[data-theme='dark'] .aa-DetachedSearchButton { - color: #a0a8b4; - border-color: rgba(160, 168, 180, 0.4); -} - -[data-theme='dark'] .aa-DetachedSearchButtonIcon, -[data-theme='dark'] .aa-DetachedSearchButtonPlaceholder { - color: #a0a8b4; -} - /* Compact inline footer */ .footer { padding: 1.5rem 2rem; diff --git a/src/plugins/changelog/theme/ChangelogPage/index.tsx b/src/plugins/changelog/theme/ChangelogPage/index.tsx index f6c9c774..7a38f9fd 100644 --- a/src/plugins/changelog/theme/ChangelogPage/index.tsx +++ b/src/plugins/changelog/theme/ChangelogPage/index.tsx @@ -31,7 +31,7 @@ function ChangelogPage(props: Props): JSX.Element {
-
+
← Back to Changelog
diff --git a/src/plugins/search/engine.d.ts b/src/plugins/search/engine.d.ts new file mode 100644 index 00000000..a7f246a6 --- /dev/null +++ b/src/plugins/search/engine.d.ts @@ -0,0 +1,45 @@ +/** Text split into [text, isMatch] pairs, for highlighting. */ +export type Highlighted = [text: string, isMatch: boolean][]; + +export type PageKind = 'docs' | 'api' | 'changelog'; + +export interface SectionResult { + url: string; + /** Null for the page itself (its introduction). */ + heading: Highlighted | null; + parents: string[]; + /** "GET /v1/entity/{slug}" for API operations. */ + endpoint: string | null; + deprecated: boolean; + /** The excerpt comes from a code sample. */ + isCode: boolean; + excerpt: { prefix: boolean; suffix: boolean; text: Highlighted } | null; + score: number; +} + +export interface PageResult { + url: string; + title: Highlighted; + kind: PageKind; + breadcrumbs: string[]; + score: number; + sections: SectionResult[]; +} + +export interface SearchResults { + pages: PageResult[]; + /** Number of matching pages, before `limit`. */ + total: number; + /** Number of matching pages per kind, regardless of the `kind` filter. */ + counts: Partial>; + terms: string[]; +} + +export interface SearchIndex { + /** Indexes for at most `budgetMs` milliseconds. Returns true when done. */ + build(budgetMs?: number): boolean; + search(query: string, options?: { kind?: PageKind; limit?: number; sectionsPerPage?: number }): SearchResults; + size: number; +} + +export function createIndex(data: { pages: unknown[]; sections: unknown[] }): SearchIndex; diff --git a/src/plugins/search/engine.js b/src/plugins/search/engine.js new file mode 100644 index 00000000..142b5ab3 --- /dev/null +++ b/src/plugins/search/engine.js @@ -0,0 +1,768 @@ +/** + * Search engine for the docs site. Plain JavaScript without dependencies: it + * runs in the browser (search modal) and in Node (relevance checks). + * + * Ranking, in order of application: + * + * 1. BM25F. Every section is a document with five fields (title, keys, + * context, body, code). Term frequencies are length-normalised per field, + * weighted, summed and only then saturated, so one hit in a title is worth + * more than many hits in a long body. + * 2. Query term expansion. Each query word matches itself, other forms of + * the same word ("authenticate" finds "authentication", for less than an + * exact match), words it is a prefix of (weighted by how much of the word + * was typed) and, only when nothing else matches, words within a small + * edit distance (typos). All expansions of one word share a single IDF + * and saturation. + * 3. Coverage. Sections that match every query word rank far above partial + * matches; a partial match on a page that covers the query as a whole is + * penalised less. + * 4. Title, page and kind boosts, then phrase and proximity boosts on the + * best candidates. + * 5. Sections are grouped into pages, each page shows its best sections. + */ + +const TITLE = 0; +const KEYS = 1; +const CONTEXT = 2; +// 3: body, 4: code +const FIELD_COUNT = 5; + +// Relative worth of one occurrence per field, and how strongly the field +// length dampens it (BM25 `b`). +const FIELD_WEIGHT = [9, 9, 2.2, 1, 0.45]; +const FIELD_B = [0.35, 0.2, 0.2, 0.75, 0.6]; +const K1 = 1.2; + +const KIND_WEIGHT = { docs: 1, api: 0.92, changelog: 0.3 }; + +const STOPWORDS = new Set( + 'a an and are as at be by can do does for from how i in is it my of on or that the this to via what when where which with you your'.split( + ' ', + ), +); + +// Title words that say nothing about the topic of a page. +const FILLER_WORDS = new Set(['overview', 'introduction', 'intro', 'readme', 'guide', 'reference', 'epilot']); + +const WORD = /[A-Za-z0-9À-ɏ]+/g; +const CAMEL_PART = /[A-Z]?[a-z]+|[A-Z]+(?![a-z])|[0-9]+/g; +const DIACRITICS = /[̀-ͯ]/g; + +const normalize = (word) => { + const lower = word.toLowerCase(); + + // eslint-disable-next-line no-control-regex + return /^[\x00-\x7f]*$/.test(lower) ? lower : lower.normalize('NFKD').replace(DIACRITICS, ''); +}; + +/** + * Folds plurals only ("entities" -> "entity", "webhooks" -> "webhook"). Words + * that differ by nothing but a plural count as the same word. + */ +const singular = (term) => { + if (term.length < 4 || !/^[a-z]+$/.test(term)) return term; + if (term.endsWith('ies')) return `${term.slice(0, -3)}y`; + if (/(ss|x|z|ch|sh)es$/.test(term)) return term.slice(0, -2); + if (term.endsWith('s') && !/(ss|us|is)$/.test(term)) return term.slice(0, -1); + + return term; +}; + +const STEP2 = { + ational: 'ate', + tional: 'tion', + enci: 'ence', + anci: 'ance', + izer: 'ize', + bli: 'ble', + alli: 'al', + entli: 'ent', + eli: 'e', + ousli: 'ous', + ization: 'ize', + ation: 'ate', + ator: 'ate', + alism: 'al', + iveness: 'ive', + fulness: 'ful', + ousness: 'ous', + aliti: 'al', + iviti: 'ive', + biliti: 'ble', + logi: 'log', +}; +const STEP3 = { icate: 'ic', ative: '', alize: 'al', iciti: 'ic', ical: 'ic', ful: '', ness: '' }; +const CONSONANTS = '[^aeiou][^aeiouy]*'; +const VOWELS = '[aeiouy][aeiou]*'; +const MEASURE_GT0 = new RegExp(`^(${CONSONANTS})?${VOWELS}${CONSONANTS}`); +const MEASURE_EQ1 = new RegExp(`^(${CONSONANTS})?${VOWELS}${CONSONANTS}(${VOWELS})?$`); +const MEASURE_GT1 = new RegExp(`^(${CONSONANTS})?${VOWELS}${CONSONANTS}${VOWELS}${CONSONANTS}`); +const HAS_VOWEL = new RegExp(`^(${CONSONANTS})?[aeiouy]`); +const ENDS_CVC = new RegExp(`^${CONSONANTS}[aeiouy][^aeiouwxy]$`); +const STEP2_SUFFIX = new RegExp(`^(.+?)(${Object.keys(STEP2).join('|')})$`); +const STEP3_SUFFIX = new RegExp(`^(.+?)(${Object.keys(STEP3).join('|')})$`); +const STEP4_SUFFIX = /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/; +const stems = new Map(); + +/** + * Porter stemmer: reduces a word to its root, so that inflections and + * derivations of a word find each other ("authenticate", "authenticated", + * "authentication"). + */ +const porter = (word) => { + if (word.length < 4 || !/^[a-z]+$/.test(word)) return word; + + let w = word; + let match; + + if (w[0] === 'y') w = `Y${w.slice(1)}`; + + // Step 1a: plurals + if (/^(.+?)(ss|i)es$/.test(w)) w = w.slice(0, -2); + else if (/^(.+?)([^s])s$/.test(w)) w = w.slice(0, -1); + + // Step 1b: -eed, -ed, -ing + if ((match = /^(.+?)eed$/.exec(w))) { + if (MEASURE_GT0.test(match[1])) w = w.slice(0, -1); + } else if ((match = /^(.+?)(ed|ing)$/.exec(w)) && HAS_VOWEL.test(match[1])) { + w = match[1]; + if (/(at|bl|iz)$/.test(w)) w += 'e'; + else if (/([^aeiouylsz])\1$/.test(w)) w = w.slice(0, -1); + else if (ENDS_CVC.test(w)) w += 'e'; + } + + // Step 1c: -y + if ((match = /^(.+?)y$/.exec(w)) && HAS_VOWEL.test(match[1])) w = `${match[1]}i`; + + // Steps 2 and 3: double and derivational suffixes + if ((match = STEP2_SUFFIX.exec(w)) && MEASURE_GT0.test(match[1])) w = match[1] + STEP2[match[2]]; + if ((match = STEP3_SUFFIX.exec(w)) && MEASURE_GT0.test(match[1])) w = match[1] + STEP3[match[2]]; + + // Step 4: remaining suffixes + if ((match = STEP4_SUFFIX.exec(w))) { + if (MEASURE_GT1.test(match[1])) w = match[1]; + } else if ((match = /^(.+?)(s|t)(ion)$/.exec(w)) && MEASURE_GT1.test(match[1] + match[2])) { + w = match[1] + match[2]; + } + + // Step 5: final -e and -ll + if ((match = /^(.+?)e$/.exec(w))) { + if (MEASURE_GT1.test(match[1]) || (MEASURE_EQ1.test(match[1]) && !ENDS_CVC.test(match[1]))) w = match[1]; + } + if (/ll$/.test(w) && MEASURE_GT1.test(w)) w = w.slice(0, -1); + + return w[0] === 'Y' ? `y${w.slice(1)}` : w; +}; + +/** + * Index key shared by all forms of a word. Prefixed, so it never collides + * with a word itself. + */ +const rootOf = (word) => { + let root = stems.get(word); + + if (root === undefined) stems.set(word, (root = `~${porter(word)}`)); + + return root; +}; + +// An exact word is worth EXACT + ROOT, another form of it only ROOT. +const EXACT = 0.6; +const ROOT = 0.4; + +/** + * Splits text into words with their offsets. An identifier such as + * `getEntityV2` yields the whole word plus its camelCase parts, so both + * "getEntity" and "get entity" find it. + */ +function tokenize(text) { + const tokens = []; + let match; + + WORD.lastIndex = 0; + + while ((match = WORD.exec(text))) { + const word = match[0]; + const terms = [normalize(word)]; + + if (/[a-z][A-Z]|[A-Z]{2}[a-z]/.test(word)) { + for (const part of word.match(CAMEL_PART) || []) { + if (part.length > 1) terms.push(normalize(part)); + } + } + + tokens.push({ terms, start: match.index, end: match.index + word.length }); + } + + return tokens; +} + +/** Optimal string alignment distance, cut off above `max`. */ +function editDistance(a, b, max) { + if (Math.abs(a.length - b.length) > max) return max + 1; + + let previous2 = null; + let previous = Array.from({ length: b.length + 1 }, (_, j) => j); + + for (let i = 1; i <= a.length; i++) { + const current = [i]; + let rowMin = i; + + for (let j = 1; j <= b.length; j++) { + const cost = a[i - 1] === b[j - 1] ? 0 : 1; + let value = Math.min(previous[j] + 1, current[j - 1] + 1, previous[j - 1] + cost); + + if (previous2 && i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]) { + value = Math.min(value, previous2[j - 2] + 1); + } + + current.push(value); + if (value < rowMin) rowMin = value; + } + + if (rowMin > max) return max + 1; + + previous2 = previous; + previous = current; + } + + return previous[b.length]; +} + +/** + * @param {{pages: Array, sections: Array}} data Contents of search-index.json + */ +function createIndex(data) { + const { pages, sections } = data; + const count = sections.length; + const postings = new Map(); // term -> [docId * FIELD_COUNT + field, tf, ...] + const vocabulary = new Set(); // unstemmed words, for prefix and typo lookup + const fieldLength = Array.from({ length: FIELD_COUNT }, () => new Float32Array(count)); + const averageLength = new Array(FIELD_COUNT).fill(1); + let sortedVocabulary = []; + let indexed = 0; + + const fieldsOf = (id) => { + const [pageIndex, heading, , parents, text, extra] = sections[id]; + const page = pages[pageIndex]; + + return [ + heading || page[1], + extra?.k || '', + [heading ? page[1] : '', ...parents, ...page[3]].join(' '), + text, + extra?.c || '', + ]; + }; + + const addDocument = (id) => { + const fields = fieldsOf(id); + + for (let field = 0; field < FIELD_COUNT; field++) { + if (!fields[field]) continue; + + const tokens = tokenize(fields[field]); + const frequencies = new Map(); + + for (const token of tokens) { + for (const word of token.terms) { + if (word.length < 2) continue; + vocabulary.add(word); + + const term = singular(word); + const root = rootOf(term); + + frequencies.set(term, (frequencies.get(term) || 0) + 1); + frequencies.set(root, (frequencies.get(root) || 0) + 1); + } + } + + fieldLength[field][id] = tokens.length; + + for (const [term, frequency] of frequencies) { + let list = postings.get(term); + + if (!list) postings.set(term, (list = [])); + list.push(id * FIELD_COUNT + field, frequency); + } + } + }; + + /** + * Indexes the next sections for at most `budgetMs`, so the browser can + * build the index in slices without blocking input. Returns true when done. + */ + const build = (budgetMs = Infinity) => { + const deadline = Date.now() + budgetMs; + + while (indexed < count) { + addDocument(indexed++); + if (indexed % 64 === 0 && Date.now() > deadline) return false; + } + + for (let field = 0; field < FIELD_COUNT; field++) { + let total = 0; + let filled = 0; + + for (let id = 0; id < count; id++) { + if (fieldLength[field][id]) { + total += fieldLength[field][id]; + filled++; + } + } + averageLength[field] = filled ? total / filled : 1; + } + + sortedVocabulary = [...vocabulary].sort(); + + return true; + }; + + /** Words of the vocabulary starting with `prefix` (binary search). */ + const wordsWithPrefix = (prefix, limit) => { + let low = 0; + let high = sortedVocabulary.length; + + while (low < high) { + const middle = (low + high) >> 1; + + if (sortedVocabulary[middle] < prefix) low = middle + 1; + else high = middle; + } + + const words = []; + + for (let i = low; i < sortedVocabulary.length && sortedVocabulary[i].startsWith(prefix); i++) { + words.push(sortedVocabulary[i]); + } + + // Shortest first: the closest completions of what was typed. + return words.sort((a, b) => a.length - b.length).slice(0, limit); + }; + + /** + * Index terms a query word may match, with a weight between 0 and 1. + */ + const expand = (word, isLast) => { + // term -> [weight in the score, closeness to what was typed (0..1)] + const candidates = new Map(); + const offer = (term, weight, closeness = weight) => { + if (postings.has(term) && weight > (candidates.get(term)?.[0] || 0)) candidates.set(term, [weight, closeness]); + }; + const own = singular(word); + + offer(own, EXACT, 1); + offer(rootOf(own), ROOT, 0.85); + + // While typing, the last word is usually incomplete. Earlier words are + // complete more often than not, so their completions count for less. + if (word.length >= (isLast ? 2 : 3)) { + for (const completion of wordsWithPrefix(word, 150)) { + const typed = word.length / completion.length; + + offer(singular(completion), (isLast ? 0.5 : 0.35) + 0.45 * typed); + } + } + + // Typos: only when the word is unknown, otherwise "entity" would also + // return everything about "entry". + if (!candidates.size && word.length >= 4) { + const max = word.length >= 8 ? 2 : 1; + + for (const known of sortedVocabulary) { + if (known[0] !== word[0] && known[1] !== word[1]) continue; + + const distance = editDistance(word, known, max); + + if (distance <= max) { + offer(singular(known), distance === 1 ? 0.6 : 0.4); + } else if (isLast && known.length > word.length && editDistance(word, known.slice(0, word.length), 1) <= 1) { + // A typo in a word that is still being typed. + offer(singular(known), 0.3); + } + } + } + + return candidates; + }; + + const parseQuery = (query) => { + const seen = new Set(); + const words = []; + + for (const token of tokenize(query)) { + const word = token.terms[0]; + + if (!seen.has(word)) { + seen.add(word); + words.push(word); + } + } + + const meaningful = words.filter((word) => word.length > 1 || words.length === 1); + const hasContentWord = meaningful.some((word) => !STOPWORDS.has(word)); + + return meaningful.slice(0, 12).map((word, index, all) => ({ + word, + isLast: index === all.length - 1 && !/\s$/.test(query), + // Stopwords still score, but a section is not required to contain them. + importance: hasContentWord && STOPWORDS.has(word) ? 0.1 : 1, + })); + }; + + /** Whether a word of the text is one of the index terms a query matched. */ + const isMatch = (word, terms) => { + if (word.length < 2) return false; + + const term = singular(word); + + return terms.has(term) || terms.has(rootOf(term)); + }; + + /** + * Highlights matching words. Returns [text, isMatch] pairs. + */ + const highlight = (text, matchTerms) => { + const segments = []; + let cursor = 0; + + for (const token of tokenize(text)) { + if (!token.terms.some((word) => isMatch(word, matchTerms))) continue; + if (token.start > cursor) segments.push([text.slice(cursor, token.start), false]); + segments.push([text.slice(token.start, token.end), true]); + cursor = token.end; + } + + if (cursor < text.length) segments.push([text.slice(cursor), false]); + + return segments; + }; + + /** + * Picks the passage of `text` that contains the most distinct query words + * closest together. + */ + const excerpt = (text, queryTerms, length = 170) => { + if (!text) return null; + + const hits = []; + + for (const token of tokenize(text)) { + let best = -1; + + for (const word of token.terms) { + for (let i = 0; i < queryTerms.length; i++) { + if (isMatch(word, queryTerms[i].candidates)) best = i; + } + } + if (best >= 0) hits.push({ query: best, start: token.start, end: token.end }); + } + + if (!hits.length) return null; + + let bestStart = hits[0].start; + let bestScore = -1; + + for (let i = 0; i < hits.length; i++) { + const covered = new Set(); + let score = 0; + + for (let j = i; j < hits.length && hits[j].end - hits[i].start <= length; j++) { + // Distinct query words count most, repeats a little. + score += covered.has(hits[j].query) ? 0.1 : queryTerms[hits[j].query].idf + 1; + covered.add(hits[j].query); + } + + if (score > bestScore) { + bestScore = score; + bestStart = hits[i].start; + } + } + + // Lead in with some context, starting at a word (ideally a sentence). + let start = Math.max(0, bestStart - 40); + + if (start > 0) { + const sentence = text.lastIndexOf('. ', bestStart); + + if (sentence >= start - 40 && sentence < bestStart) start = sentence + 2; + else start = text.indexOf(' ', start) + 1 || start; + } + + let end = Math.min(text.length, start + length); + + if (end < text.length) end = text.lastIndexOf(' ', end) > start ? text.lastIndexOf(' ', end) : end; + + return { + prefix: start > 0, + suffix: end < text.length, + text: text.slice(start, end), + }; + }; + + /** + * Smallest distance, in words, over which the query words appear in order + * of the text. 1 per word means an exact phrase. + */ + const proximity = (text, queryTerms) => { + const positions = queryTerms.map(() => []); + + tokenize(text).forEach((token, position) => { + for (const word of token.terms) { + queryTerms.forEach((queryTerm, i) => { + if (isMatch(word, queryTerm.candidates) && !positions[i].includes(position)) positions[i].push(position); + }); + } + }); + + const present = positions.filter((list) => list.length); + + if (present.length < 2) return { phrase: false, span: Infinity, words: present.length }; + + // Phrase: every present word directly follows the previous one. + const phrase = present[0].some((position) => present.every((list, offset) => list.includes(position + offset))); + + let span = Infinity; + + for (const anchor of present[0]) { + let low = anchor; + let high = anchor; + + for (let i = 1; i < present.length; i++) { + let nearest = present[i][0]; + + for (const position of present[i]) { + if (Math.abs(position - anchor) < Math.abs(nearest - anchor)) nearest = position; + } + low = Math.min(low, nearest); + high = Math.max(high, nearest); + } + span = Math.min(span, high - low + 1); + } + + return { phrase: phrase && present.length === queryTerms.length, span, words: present.length }; + }; + + /** + * @param {string} query + * @param {object} [options] + * @param {string} [options.kind] Only pages of this kind (docs, api, changelog) + * @param {number} [options.limit] Maximum number of pages + * @param {number} [options.sectionsPerPage] Maximum sections shown per page + */ + const search = (query, { kind, limit = 8, sectionsPerPage = 3 } = {}) => { + const queryTerms = parseQuery(query).map((term) => ({ ...term, candidates: expand(term.word, term.isLast) })); + const empty = { pages: [], total: 0, counts: {}, terms: [] }; + + if (!queryTerms.length) return empty; + + const scores = new Float32Array(count); + const matched = new Int32Array(count); // bitmask: query words found anywhere + const matchedOwn = new Int32Array(count); // ... found outside the context field + // Per query word: how closely the title or keys of a section match it. + const titleMatch = queryTerms.map(() => new Float32Array(count)); + + queryTerms.forEach((queryTerm, index) => { + const bit = 1 << index; + const weighted = new Map(); // docId -> weighted, normalised term frequency + + for (const [term, [weight, closeness]] of queryTerm.candidates) { + const list = postings.get(term); + + for (let i = 0; i < list.length; i += 2) { + const field = list[i] % FIELD_COUNT; + const id = (list[i] - field) / FIELD_COUNT; + const norm = 1 - FIELD_B[field] + (FIELD_B[field] * fieldLength[field][id]) / averageLength[field]; + + weighted.set(id, (weighted.get(id) || 0) + (weight * FIELD_WEIGHT[field] * list[i + 1]) / norm); + matched[id] |= bit; + if (field !== CONTEXT) matchedOwn[id] |= bit; + if ((field === TITLE || field === KEYS) && closeness > titleMatch[index][id]) + titleMatch[index][id] = closeness; + } + } + + queryTerm.idf = Math.log(1 + (count - weighted.size + 0.5) / (weighted.size + 0.5)); + + for (const [id, frequency] of weighted) { + scores[id] += queryTerm.importance * queryTerm.idf * (frequency / (K1 + frequency)) * (K1 + 1); + } + }); + + const allBits = (1 << queryTerms.length) - 1; + const requiredBits = queryTerms.reduce((bits, term, i) => (term.importance === 1 ? bits | (1 << i) : bits), 0); + const totalImportance = queryTerms.reduce((sum, term) => sum + term.importance * (term.idf + 0.5), 0); + const coverageOf = (bits) => { + if (bits === allBits) return 1; + + let sum = 0; + + queryTerms.forEach((term, i) => { + if (bits & (1 << i)) sum += term.importance * (term.idf + 0.5); + }); + + return sum / totalImportance; + }; + + // What the page as a whole covers, to go easier on sections of a page + // that answers the query across several sections. + const pageBits = new Map(); + + for (let id = 0; id < count; id++) { + if (matched[id]) pageBits.set(sections[id][0], (pageBits.get(sections[id][0]) || 0) | matched[id]); + } + + const joinedQuery = queryTerms.map((term) => term.word).join(''); + const pagesByKind = {}; + let candidates = []; + + for (let id = 0; id < count; id++) { + // A section that only matches through its page title or parent + // headings is not a result by itself. + if (!matchedOwn[id]) continue; + + const section = sections[id]; + const page = pages[section[0]]; + + // Counted before filtering, so the filters can show what each holds. + (pagesByKind[page[2]] = pagesByKind[page[2]] || new Set()).add(section[0]); + if (kind && page[2] !== kind) continue; + + const coverage = coverageOf(matched[id]); + const pageCoverage = coverageOf(pageBits.get(section[0])); + let score = scores[id] * Math.max(coverage ** 3, 0.45 * coverage * pageCoverage ** 3); + + // How well the title covers the query: the weakest required word decides. + let inTitle = requiredBits ? 1 : 0; + + queryTerms.forEach((term, i) => { + if (term.importance === 1) inTitle = Math.min(inTitle, titleMatch[i][id]); + }); + + if (inTitle > 0) { + // Every query word is in the title: this section is about the query. + // The less else the title says, the more: "Webhooks" is a better + // answer to "webhooks" than "Environments & Secrets in Webhooks". + // An identifier counts by its parts: "pricing" is a quarter of + // "PricingEntitiesExtractResult". + let words = 0; + let matching = 0; + + for (const token of tokenize(section[1] || page[1])) { + if (STOPWORDS.has(token.terms[0]) || FILLER_WORDS.has(token.terms[0])) continue; + + const parts = token.terms.length > 1 ? token.terms.slice(1) : token.terms; + const isWhole = queryTerms.some((term) => isMatch(token.terms[0], term.candidates)); + + words += parts.length; + matching += isWhole + ? parts.length + : parts.filter((word) => queryTerms.some((term) => isMatch(word, term.candidates))).length; + } + + // An operation is named by its operationId, the first of its keys. + const isKey = Boolean(section[5]?.e) && section[5].k.split(' ')[0].toLowerCase() === joinedQuery; + let boost = 1.5 * (1 + 0.6 * (isKey ? 1 : matching / Math.max(words, queryTerms.length))); + + // The page about a topic beats a section about it somewhere else. + if (!section[1]) boost *= 1.8; + + score *= 1 + (boost - 1) * inTitle; + } + + score *= (KIND_WEIGHT[page[2]] || 1) * (page[4] || 1); + if (section[5]?.d) score *= 0.5; + + candidates.push({ id, score }); + } + + candidates.sort((a, b) => b.score - a.score); + + // Phrase and proximity need the text, so only the best candidates get them. + if (queryTerms.length > 1) { + const head = candidates.slice(0, 120); + + for (const candidate of head) { + const section = sections[candidate.id]; + const page = pages[section[0]]; + const inTitle = proximity(`${section[1] || page[1]} ${section[5]?.k || ''}`, queryTerms); + const inBody = proximity(section[4], queryTerms); + + if (inTitle.phrase) candidate.score *= 1.5; + else if (inBody.phrase) candidate.score *= 1.35; + else if (inBody.words === queryTerms.length) + candidate.score *= 1 + 0.3 * Math.min(1, queryTerms.length / inBody.span); + } + + candidates = head.sort((a, b) => b.score - a.score).concat(candidates.slice(120)); + } + + // Group into pages. Further matching sections add a little to the page. + const groups = new Map(); + + for (const candidate of candidates) { + const pageIndex = sections[candidate.id][0]; + let group = groups.get(pageIndex); + + if (!group) groups.set(pageIndex, (group = { pageIndex, score: 0, hits: [] })); + if (group.hits.length < 6) group.score += candidate.score * (group.hits.length ? 0.12 / group.hits.length : 1); + group.hits.push(candidate); + } + + const ranked = [...groups.values()].sort((a, b) => b.score - a.score); + const counts = {}; + + for (const [pageKind, found] of Object.entries(pagesByKind)) counts[pageKind] = found.size; + + const matchTerms = new Set(); + + for (const queryTerm of queryTerms) for (const term of queryTerm.candidates.keys()) matchTerms.add(term); + + // Weak tail: results far below the best one are noise. + const threshold = ranked.length ? ranked[0].score * 0.04 : 0; + + const results = ranked + .filter((group) => group.score >= threshold) + .slice(0, limit) + .map((group) => { + const [url, title, pageKind, breadcrumbs] = pages[group.pageIndex]; + const best = group.hits[0].score; + + return { + url, + title: highlight(title, matchTerms), + kind: pageKind, + breadcrumbs, + score: group.score, + sections: group.hits + .filter((hit, index) => index === 0 || hit.score >= best * 0.2) + .slice(0, sectionsPerPage) + .map((hit) => { + const [, heading, anchor, parents, text, extra] = sections[hit.id]; + const fromText = excerpt(text, queryTerms); + const fromCode = fromText ? null : excerpt(extra?.c, queryTerms); + const passage = fromText || + fromCode || { prefix: false, suffix: text.length > 170, text: text.slice(0, 170) }; + + return { + url: anchor ? `${url}#${anchor}` : url, + heading: heading ? highlight(heading, matchTerms) : null, + parents, + endpoint: extra?.e || null, + deprecated: Boolean(extra?.d), + isCode: Boolean(fromCode), + excerpt: passage.text ? { ...passage, text: highlight(passage.text, matchTerms) } : null, + score: hit.score, + }; + }), + }; + }); + + return { pages: results, total: ranked.length, counts, terms: queryTerms.map((term) => term.word) }; + }; + + return { build, search, size: count }; +} + +module.exports = { createIndex, tokenize, singular, porter, editDistance }; diff --git a/src/plugins/search/index.js b/src/plugins/search/index.js new file mode 100644 index 00000000..f5daa5e7 --- /dev/null +++ b/src/plugins/search/index.js @@ -0,0 +1,90 @@ +/* eslint-disable @typescript-eslint/no-var-requires */ +/** + * Site search. + * + * - `indexer.js` runs after the build and writes `search-index.json`: one + * record per section of every doc, changelog entry and API operation. + * - `engine.js` ranks those records. It runs in the browser. + * - `theme/SearchBar` is the search box and results dialog in the navbar. + * + * `npm run test:search` verifies the ranking against the last build. + */ +const path = require('path'); + +const fs = require('fs-extra'); + +const { buildSearchIndex } = require('./indexer'); + +const PLUGIN_NAME = 'docusaurus-plugin-search'; +const INDEX_FILENAME = 'search-index.json'; + +const DEFAULT_OPTIONS = { + // API reference pages rendered by Redoc: [{ routePath, specUrl, layout: { title } }] + specs: [], + // Routes that are listings or navigation, not content. + exclude: [/^\/$/, /^\/(404|search)\//, /^\/blog\//, /^\/api\/changelog\/(page\/.*)?$/], + // Generated reference pages repeat what the guides and the API reference + // say. They are searchable, but rank below both. + weights: [ + { test: /^\/docs\/(sdk\/clients|cli\/commands)\//, weight: 0.7 }, + { test: /-library\//, weight: 0.7 }, + ], +}; + +module.exports = function searchPlugin(context, pluginOptions) { + const options = { ...DEFAULT_OPTIONS, ...pluginOptions }; + // The dev server has no built pages to index. It serves the index of the + // last production build instead, so search can be tried out locally. + const devDir = path.join(context.generatedFilesDir, PLUGIN_NAME); + + return { + name: PLUGIN_NAME, + + getThemePath() { + return path.resolve(__dirname, './theme'); + }, + + async loadContent() { + const lastBuild = path.join(context.outDir, INDEX_FILENAME); + + await fs.ensureDir(devDir); + if (process.env.NODE_ENV !== 'production' && (await fs.pathExists(lastBuild))) { + await fs.copy(lastBuild, path.join(devDir, INDEX_FILENAME)); + } + }, + + configureWebpack(_config, isServer) { + if (isServer || process.env.NODE_ENV === 'production') return {}; + + return { + devServer: { + static: [{ publicPath: context.siteConfig.baseUrl, directory: devDir, watch: false }], + }, + }; + }, + + async postBuild({ outDir, routesPaths, baseUrl, siteConfig }) { + const started = Date.now(); + const index = await buildSearchIndex({ + outDir, + routesPaths, + baseUrl, + siteTitle: siteConfig.title, + specs: options.specs, + exclude: (route) => options.exclude.some((pattern) => pattern.test(route)), + weightOf: (route) => options.weights.find(({ test }) => test.test(route))?.weight ?? 1, + log: (message) => console.warn(`[${PLUGIN_NAME}] ${message}`), + }); + const json = JSON.stringify(index); + + await fs.writeFile(path.join(outDir, INDEX_FILENAME), json, 'utf-8'); + + console.log( + `[${PLUGIN_NAME}] Indexed ${index.sections.length} sections of ${index.pages.length} pages ` + + `(${(json.length / 1024 / 1024).toFixed(1)} MB) in ${((Date.now() - started) / 1000).toFixed(1)}s.`, + ); + }, + }; +}; + +module.exports.INDEX_FILENAME = INDEX_FILENAME; diff --git a/src/plugins/search/indexer.js b/src/plugins/search/indexer.js new file mode 100644 index 00000000..e4ec0c0b --- /dev/null +++ b/src/plugins/search/indexer.js @@ -0,0 +1,453 @@ +/* eslint-disable @typescript-eslint/no-var-requires */ +/** + * Build-time half of the site search: turns the built site into a list of + * searchable records, one per page section. + * + * - Doc, changelog and custom pages are read from the rendered HTML, so the + * index contains exactly what a reader sees (including MDX components) and + * the anchors are the real heading ids. + * - API reference pages are rendered client-side by Redoc and have no content + * in the HTML, so their operations are read from the OpenAPI specs instead. + */ +const path = require('path'); + +const cheerio = require('cheerio'); +const fs = require('fs-extra'); +const yaml = require('yaml'); + +// h5 and h6 are admonition titles and other labels, not places to link to. +const HEADINGS = new Set(['h1', 'h2', 'h3', 'h4']); + +// Never descended into: chrome, controls and rendered diagrams. Anything else +// can opt out with a `data-search-ignore` attribute. +const SKIPPED_TAGS = new Set(['script', 'style', 'noscript', 'svg', 'button', 'nav', 'footer', 'iframe', 'template']); +const SKIPPED_CLASSES = [ + 'hash-link', + 'mermaid', + 'theme-doc-toc-mobile', + 'theme-doc-footer', + 'pagination-nav', + 'theme-edit-this-page', + 'sr-only', +]; + +// Elements that end a run of text. Everything else is inline. +const BLOCK_TAGS = new Set([ + 'p', + 'div', + 'section', + 'article', + 'header', + 'ul', + 'ol', + 'li', + 'dl', + 'dt', + 'dd', + 'table', + 'thead', + 'tbody', + 'tr', + 'td', + 'th', + 'pre', + 'blockquote', + 'details', + 'summary', + 'figure', + 'figcaption', + 'br', + 'hr', +]); + +const ADMONITION_LABELS = new Set(['note', 'tip', 'info', 'caution', 'danger', 'warning', 'important', 'success']); + +const GENERIC_TITLES = new Set(['overview', 'introduction', 'intro', 'readme', 'getting started']); + +const HTTP_METHODS = ['get', 'post', 'put', 'patch', 'delete', 'options', 'head']; + +// Long sections are cut: nobody reads past this in a search result, and the +// tail of a huge generated table only adds noise to the ranking. +const MAX_SECTION_CHARS = 6000; +const MAX_API_DESCRIPTION_CHARS = 1500; + +// Code samples are searchable but kept apart from the prose, so the engine can +// weight them lower: a 200-line sample response must not outrank the paragraph +// that explains the feature. Only the head of a block is kept. +const MAX_CODE_BLOCK_CHARS = 600; +const MAX_SECTION_CODE_CHARS = 1800; + +const clean = (text) => + (text || '') + .replace(/\u200b|\u00ad/g, '') + .replace(/\s+/g, ' ') + .trim(); + +const hasSkippedClass = (el) => { + const classes = el.attribs?.class; + + return Boolean(classes) && SKIPPED_CLASSES.some((name) => classes.split(/\s+/).includes(name)); +}; + +/** + * Walks a rendered page in document order and splits it at every heading. + * A heading without an id belongs to the nearest anchor above it, so its + * result still links to a place close to the text. + */ +function splitIntoSections($, root) { + const sections = [{ heading: null, level: 0, anchor: '', parts: [], code: [] }]; + let current = sections[0]; + let lastAnchor = ''; + + const walk = (node) => { + if (node.type === 'text') { + current.parts.push(node.data); + + return; + } + if (node.type !== 'tag') return; + + const tag = node.name; + + if (SKIPPED_TAGS.has(tag) || hasSkippedClass(node) || 'data-search-ignore' in node.attribs) return; + + // "note", "tip" and the like label the box, they are not part of the text. + if (/\badmonition-heading\b/.test(node.attribs.class || '')) { + if (ADMONITION_LABELS.has(clean($(node).text()).toLowerCase())) return; + } + + if (tag === 'pre') { + const code = clean($(node).text()).slice(0, MAX_CODE_BLOCK_CHARS); + + if (code) current.code.push(code); + + return; + } + + if (HEADINGS.has(tag)) { + const heading = $(node).clone(); + + heading.find('.hash-link, button, svg').remove(); + + const text = clean(heading.text()); + + if (!text) return; + if (node.attribs.id) lastAnchor = node.attribs.id; + + current = { heading: text, level: Number(tag[1]), anchor: lastAnchor, parts: [], code: [] }; + sections.push(current); + + return; + } + + const isBlock = BLOCK_TAGS.has(tag) || (tag === 'a' && /\bbutton\b/.test(node.attribs.class || '')); + + if (isBlock) current.parts.push('\n'); + node.children.forEach(walk); + if (isBlock) current.parts.push('\n'); + }; + + walk(root); + + return sections.map(({ parts, code, ...section }) => ({ + ...section, + text: clean(parts.join('')).slice(0, MAX_SECTION_CHARS), + code: code.join(' \n ').slice(0, MAX_SECTION_CODE_CHARS), + })); +} + +const kindOfRoute = (route) => { + if (route.startsWith('/api/changelog/')) return 'changelog'; + if (route.startsWith('/api/')) return 'api'; + + return 'docs'; +}; + +/** + * Reads one built page. Returns null for anything that is not content: + * redirect stubs, listings, pages marked noindex. + */ +function readHtmlPage(html, route, siteTitle) { + const $ = cheerio.load(html); + + if ($('meta[http-equiv="refresh"]').length) return null; + if (/noindex/i.test($('meta[name="robots"]').attr('content') || '')) return null; + + const root = $('.theme-doc-markdown').get(0) || $('article').get(0) || $('main').get(0); + + if (!root) return null; + + const sections = splitIntoSections($, root); + const metaTitle = clean($('title').first().text()).replace(new RegExp(`\\s*\\|\\s*${siteTitle}$`), ''); + const h1 = sections.find((section) => section.level === 1); + const title = h1?.heading || metaTitle; + + if (!title) return null; + + // Sidebar position of the page, e.g. Integrations > Webhooks. + const breadcrumbs = $('.menu__link--active') + .map((_, el) => clean($(el).text())) + .get() + .slice(0, -1) // the page itself + .filter((label) => label && label !== title); + + // "Overview" says nothing without its section: name the page after both. + const displayTitle = + GENERIC_TITLES.has(title.toLowerCase()) && breadcrumbs.length + ? `${breadcrumbs[breadcrumbs.length - 1]} ${title}` + : title; + + const description = clean($('meta[name="description"]').attr('content')); + const records = []; + const parents = []; + + // Text above the first heading and under the h1 both describe the page itself. + const lead = sections.filter((section) => section.level <= 1); + const intro = lead + .map((section) => section.text) + .filter(Boolean) + .join(' '); + + // The URL names the page too, often better than its title + // (/docs/portals/customer-portal/ is titled "Overview"). The landing page + // of a section (/docs/sdk/overview/) is named after the section. + const segments = route.split('/').filter(Boolean); + const isLanding = GENERIC_TITLES.has((segments[segments.length - 1] || '').replace(/-/g, ' ')); + const slug = (isLanding ? segments[segments.length - 2] : segments[segments.length - 1]) || ''; + + records.push({ + heading: null, + anchor: '', + parents: [], + keys: slug.replace(/[-_]/g, ' '), + text: intro || description, + code: lead + .map((section) => section.code) + .filter(Boolean) + .join(' \n '), + }); + + for (const section of sections) { + if (section.level <= 1) continue; + + while (parents.length && parents[parents.length - 1].level >= section.level) parents.pop(); + + records.push({ + heading: section.heading, + anchor: section.anchor, + parents: parents.map((parent) => parent.heading), + text: section.text, + code: section.code, + }); + parents.push(section); + } + + return { + page: { url: route, title: displayTitle, kind: kindOfRoute(route), breadcrumbs }, + // A heading with nothing under it is still a valid target. + sections: records.filter((record, index) => index === 0 || record.heading), + }; +} + +const markdownToText = (markdown) => + clean( + (markdown || '') + .replace(/```[\s\S]*?```/g, ' ') + .replace(/<[^>]+>/g, ' ') + .replace(/!\[[^\]]*\]\([^)]*\)/g, ' ') + .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1') + .replace(/^\s*(#{1,6}|>|[-*+]|\d+\.)\s+/gm, '') + .replace(/[*_`|]/g, ' '), + ); + +/** + * Mirrors Redoc's anchor ids, so a result opens the API page scrolled to the + * operation: `#tag//operation/`. + */ +const operationAnchor = (tag, operationId, apiPath, method) => { + const prefix = tag ? `tag/${tag.replace(/\s/g, '-')}/` : ''; + + if (operationId) return `${prefix}operation/${operationId}`; + + return `${prefix}paths/${apiPath.replace(/~/g, '~0').replace(/\//g, '~1')}/${method}`; +}; + +/** + * Turns an OpenAPI document into one page (the API) and one section per + * operation. The endpoint and operationId go into `keys`, which the search + * engine weights like a title: developers search for both. + */ +function readOpenApiSpec(spec, { routePath, title }) { + const url = routePath.endsWith('/') ? routePath : `${routePath}/`; + const sections = [ + { + heading: null, + anchor: '', + parents: [], + text: markdownToText(spec.info?.description).slice(0, MAX_API_DESCRIPTION_CHARS), + }, + ]; + + for (const tag of spec.tags || []) { + const text = markdownToText(tag.description); + + if (!tag.name || !text) continue; + + sections.push({ + heading: tag['x-displayName'] || tag.name, + anchor: `tag/${tag.name.replace(/\s/g, '-')}`, + parents: [], + text: text.slice(0, MAX_API_DESCRIPTION_CHARS), + }); + } + + for (const [apiPath, pathItem] of Object.entries(spec.paths || {})) { + for (const method of HTTP_METHODS) { + const operation = pathItem?.[method]; + + if (!operation || operation['x-internal']) continue; + + const tag = operation.tags?.[0]; + const endpoint = `${method.toUpperCase()} ${apiPath}`; + const parameters = [...(pathItem.parameters || []), ...(operation.parameters || [])] + .map((parameter) => parameter?.name) + .filter(Boolean); + const description = markdownToText(operation.description).slice(0, MAX_API_DESCRIPTION_CHARS); + + sections.push({ + heading: clean(operation.summary) || operation.operationId || endpoint, + anchor: operationAnchor(tag, operation.operationId, apiPath, method), + parents: tag ? [tag] : [], + keys: [operation.operationId, endpoint].filter(Boolean).join(' '), + endpoint, + deprecated: operation.deprecated || undefined, + text: [description, parameters.length ? `Parameters: ${[...new Set(parameters)].join(', ')}` : ''] + .filter(Boolean) + .join(' '), + }); + } + } + + return { page: { url, title: title || spec.info?.title, kind: 'api', breadcrumbs: ['REST API'] }, sections }; +} + +async function fetchSpec(specUrl, timeoutMs) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + + try { + const response = await fetch(specUrl, { signal: controller.signal }); + + if (!response.ok) throw new Error(`HTTP ${response.status}`); + + return yaml.parse(await response.text()); + } finally { + clearTimeout(timer); + } +} + +async function mapWithConcurrency(items, limit, fn) { + const results = new Array(items.length); + let next = 0; + + await Promise.all( + Array.from({ length: Math.min(limit, items.length) }, async () => { + while (next < items.length) { + const index = next++; + + results[index] = await fn(items[index]); + } + }), + ); + + return results; +} + +/** + * Builds the search index for a built site. + * + * @param {object} options + * @param {string} options.outDir Build output directory + * @param {string[]} options.routesPaths Every route of the site + * @param {string} options.baseUrl Site base URL + * @param {string} options.siteTitle Site title, stripped from page titles + * @param {{routePath: string, specUrl: string, layout?: {title?: string}}[]} options.specs API reference pages + * @param {(route: string) => boolean} options.exclude Routes left out of the index + * @param {(route: string) => number} options.weightOf Ranking weight of a page, 1 by default + */ +async function buildSearchIndex({ + outDir, + routesPaths, + baseUrl, + siteTitle, + specs = [], + exclude, + weightOf = () => 1, + log = console.warn, +}) { + const pages = []; + const sections = []; + const add = (entry) => { + const pageIndex = pages.length; + + const weight = weightOf(entry.page.url); + const page = [entry.page.url, entry.page.title, entry.page.kind, entry.page.breadcrumbs]; + + if (weight !== 1) page.push(weight); + pages.push(page); + + for (const section of entry.sections) { + const extra = {}; + + if (section.code) extra.c = section.code; + if (section.keys) extra.k = section.keys; + if (section.endpoint) extra.e = section.endpoint; + if (section.deprecated) extra.d = 1; + + const record = [pageIndex, section.heading || '', section.anchor || '', section.parents, section.text || '']; + + if (Object.keys(extra).length) record.push(extra); + sections.push(record); + } + }; + + const apiRoutes = new Set(specs.map((spec) => spec.routePath.replace(/\/$/, ''))); + const routes = [...new Set(routesPaths)] + .filter((route) => route.startsWith(baseUrl)) + .map((route) => `/${route.slice(baseUrl.length)}`) + .filter((route) => !apiRoutes.has(route.replace(/\/$/, '')) && !exclude(route)) + .sort(); + + for (const route of routes) { + const file = path.join(outDir, route, 'index.html'); + + if (!(await fs.pathExists(file))) continue; + + const entry = readHtmlPage(await fs.readFile(file, 'utf-8'), route.endsWith('/') ? route : `${route}/`, siteTitle); + + if (entry) add(entry); + } + + const failed = []; + const apiEntries = await mapWithConcurrency(specs, 8, async (spec) => { + try { + return readOpenApiSpec(await fetchSpec(spec.specUrl, 20000), { + routePath: spec.routePath, + title: spec.layout?.title, + }); + } catch (error) { + failed.push(`${spec.specUrl} (${error.message})`); + + // The API stays findable by name even when its spec cannot be read. + return readOpenApiSpec({}, { routePath: spec.routePath, title: spec.layout?.title }); + } + }); + + apiEntries.forEach(add); + + if (failed.length) log(`Could not index operations of ${failed.length} API spec(s): ${failed.join(', ')}`); + + return { version: 1, pages, sections }; +} + +module.exports = { buildSearchIndex, readHtmlPage, readOpenApiSpec }; diff --git a/src/plugins/search/theme/SearchBar/index.tsx b/src/plugins/search/theme/SearchBar/index.tsx new file mode 100644 index 00000000..d8c349b5 --- /dev/null +++ b/src/plugins/search/theme/SearchBar/index.tsx @@ -0,0 +1,469 @@ +import { useHistory } from '@docusaurus/router'; +import useBaseUrl from '@docusaurus/useBaseUrl'; +import clsx from 'clsx'; +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { createPortal } from 'react-dom'; + +import { createIndex } from '../../engine'; +import type { Highlighted, PageKind, PageResult, SearchIndex, SectionResult } from '../../engine'; + +import styles from './styles.module.css'; + +const KINDS: { kind: PageKind | undefined; label: string }[] = [ + { kind: undefined, label: 'All' }, + { kind: 'docs', label: 'Docs' }, + { kind: 'api', label: 'API reference' }, + { kind: 'changelog', label: 'Changelog' }, +]; + +const KIND_LABEL: Record = { docs: 'Docs', api: 'API', changelog: 'Changelog' }; + +const SUGGESTIONS = ['webhooks', 'access tokens', 'getEntity', 'journey embed', 'entity mapping']; + +let indexPromise: Promise | null = null; + +/** + * Downloads the index once per visit and builds it in short slices, so + * typing stays responsive while it loads. + */ +function loadIndex(url: string): Promise { + if (!indexPromise) { + indexPromise = fetch(url, { cache: 'no-cache' }) + .then((response) => { + if (!response.ok) throw new Error(`HTTP ${response.status}`); + + return response.json(); + }) + .then( + (data) => + new Promise((resolve) => { + const index = createIndex(data); + const step = () => (index.build(12) ? resolve(index) : setTimeout(step, 0)); + + step(); + }), + ) + .catch((error) => { + indexPromise = null; + throw error; + }); + } + + return indexPromise; +} + +const Marked = ({ text }: { text: Highlighted }) => ( + <>{text.map(([part, isMatch], i) => (isMatch ? {part} : part))} +); + +const SearchIcon = () => ( + +); + +interface Item { + key: string; + url: string; + page: PageResult; + /** Undefined for the row of the page itself. */ + section?: SectionResult; +} + +/** + * Rows in display order: every page, followed by its matching sections. + */ +function toItems(pages: PageResult[]): { items: Item[]; preferred: number } { + const items: Item[] = []; + let preferred = 0; + + pages.forEach((page, pageIndex) => { + items.push({ key: page.url, url: page.url, page }); + + page.sections.forEach((section, sectionIndex) => { + if (!section.heading) return; + + // The best result is a section of the first page: start there. + if (pageIndex === 0 && sectionIndex === 0) preferred = items.length; + items.push({ key: `${section.url} ${sectionIndex}`, url: section.url, page, section }); + }); + }); + + return { items, preferred }; +} + +const Excerpt = ({ section }: { section: SectionResult }) => + section.excerpt ? ( + + {section.excerpt.prefix && '… '} + + {section.excerpt.suffix && ' …'} + + ) : null; + +const Endpoint = ({ endpoint }: { endpoint: string }) => { + const [method, ...path] = endpoint.split(' '); + + return ( + + {method} + {path.join(' ')} + + ); +}; + +interface DialogProps { + indexUrl: string; + onClose: () => void; +} + +function SearchDialog({ indexUrl, onClose }: DialogProps) { + const history = useHistory(); + const baseUrl = useBaseUrl('/'); + const [query, setQuery] = useState(''); + const [kind, setKind] = useState(undefined); + const [index, setIndex] = useState(null); + const [failed, setFailed] = useState(false); + const [selected, setSelected] = useState(0); + const dialog = useRef(null); + const input = useRef(null); + const list = useRef(null); + + useEffect(() => { + let active = true; + + loadIndex(indexUrl).then( + (loaded) => active && setIndex(loaded), + () => active && setFailed(true), + ); + + return () => { + active = false; + }; + }, [indexUrl]); + + // The page behind the dialog must not scroll. + useEffect(() => { + const { overflow } = document.body.style; + + document.body.style.overflow = 'hidden'; + input.current?.focus(); + + return () => { + document.body.style.overflow = overflow; + }; + }, []); + + // A single character matches everything and nothing. + const hasQuery = query.trim().length > 1; + const results = useMemo( + () => (index && hasQuery ? index.search(query, { kind, limit: 12 }) : null), + [index, query, kind, hasQuery], + ); + const { items, preferred } = useMemo(() => toItems(results?.pages ?? []), [results]); + + useEffect(() => { + setSelected(preferred); + if (list.current) list.current.scrollTop = 0; + }, [results, preferred]); + + const open = useCallback( + (url: string) => { + onClose(); + history.push(baseUrl.replace(/\/$/, '') + url); + }, + [history, baseUrl, onClose], + ); + + const select = (next: number) => { + setSelected(next); + list.current?.querySelector(`[data-index="${next}"]`)?.scrollIntoView({ block: 'nearest' }); + }; + + const onKeyDown = (event: React.KeyboardEvent) => { + if (event.key === 'Escape') { + event.preventDefault(); + onClose(); + } else if (event.key === 'ArrowDown' && items.length) { + event.preventDefault(); + select((selected + 1) % items.length); + } else if (event.key === 'ArrowUp' && items.length) { + event.preventDefault(); + select((selected - 1 + items.length) % items.length); + } else if (event.key === 'Enter' && items[selected]) { + event.preventDefault(); + + const url = baseUrl.replace(/\/$/, '') + items[selected].url; + + if (event.metaKey || event.ctrlKey) window.open(url, '_blank', 'noopener'); + else open(items[selected].url); + } else if (event.key === 'Tab') { + // Keep the focus inside the dialog. + const focusable = Array.from(dialog.current?.querySelectorAll('input, button') ?? []); + const position = focusable.indexOf(document.activeElement as HTMLElement); + const next = (position + (event.shiftKey ? -1 : 1) + focusable.length) % focusable.length; + + event.preventDefault(); + focusable[next]?.focus(); + } + }; + + const onLinkClick = (event: React.MouseEvent, url: string) => { + // Let the browser open new tabs and windows itself. + if (event.metaKey || event.ctrlKey || event.shiftKey || event.button !== 0) return; + event.preventDefault(); + open(url); + }; + + const row = (item: Item, position: number) => { + const { page, section } = item; + const intro = section ? undefined : page.sections.find((candidate) => !candidate.heading); + + return ( + onLinkClick(event, item.url)} + onMouseMove={() => position !== selected && setSelected(position)} + > + {section ? ( + <> + + + + + + {section.deprecated && Deprecated} + {section.endpoint && } + + + + ) : ( + <> + + {KIND_LABEL[page.kind]} + + + + {page.breadcrumbs.length > 0 && ( + {page.breadcrumbs.join(' › ')} + )} + + {intro && } + + )} + + ); + }; + + let body: React.ReactNode; + + if (failed) { + body = ( +

+ {process.env.NODE_ENV === 'production' + ? 'Search is unavailable right now. Reload the page to try again.' + : 'No search index yet. Run "npm run build" once: the dev server uses the index of the last build.'} +

+ ); + } else if (!hasQuery) { + body = ( +
+

Search guides, API operations and the changelog.

+
+ {SUGGESTIONS.map((suggestion) => ( + + ))} +
+
+ ); + } else if (!index) { + body =

Loading search index…

; + } else if (!items.length) { + body = ( +

+ No results for “{query.trim()}” + {kind ? ` in ${KINDS.find((entry) => entry.kind === kind)?.label}` : ''}. Check the spelling or use fewer words. +

+ ); + } else { + let position = 0; + + body = results?.pages.map((page) => { + const rows = items.filter((item) => item.page === page); + + return ( +
+ {rows.map((item) => row(item, position++))} +
+ ); + }); + } + + const counts = results?.counts ?? {}; + const totalCount = Object.values(counts).reduce((sum, count) => sum + (count ?? 0), 0); + + return ( +
event.target === event.currentTarget && onClose()}> +
+
+ + 0} + aria-controls="search-results" + aria-activedescendant={items.length ? `search-result-${selected}` : undefined} + aria-autocomplete="list" + aria-label="Search the documentation" + placeholder="Search docs and APIs" + autoComplete="off" + autoCorrect="off" + autoCapitalize="off" + spellCheck={false} + value={query} + onChange={(event) => setQuery(event.target.value)} + /> + +
+ +
+ {KINDS.map((entry) => { + const count = entry.kind ? counts[entry.kind] ?? 0 : totalCount; + + return ( + + ); + })} +
+ +
+ {body} +
+ +
+ + ↑ + ↓ to navigate + + + ↵ to open + + + Esc to close + +
+
+
+ ); +} + +export default function SearchBar(): JSX.Element { + const indexUrl = useBaseUrl('/search-index.json'); + const [isOpen, setIsOpen] = useState(false); + const [shortcut, setShortcut] = useState(''); + const button = useRef(null); + + const close = useCallback(() => { + setIsOpen(false); + button.current?.focus(); + }, []); + + useEffect(() => { + setShortcut(/Mac|iPhone|iPad/.test(navigator.platform) ? '⌘K' : 'Ctrl K'); + + const onKeyDown = (event: KeyboardEvent) => { + const target = event.target as HTMLElement | null; + const isTyping = target?.isContentEditable || /^(INPUT|TEXTAREA|SELECT)$/.test(target?.tagName ?? ''); + + if ((event.key === 'k' || event.key === 'K') && (event.metaKey || event.ctrlKey) && !event.altKey) { + event.preventDefault(); + setIsOpen((value) => !value); + } else if (event.key === '/' && !isTyping && !event.metaKey && !event.ctrlKey && !event.altKey) { + event.preventDefault(); + setIsOpen(true); + } + }; + + window.addEventListener('keydown', onKeyDown); + + return () => window.removeEventListener('keydown', onKeyDown); + }, []); + + // Start downloading the index as soon as a search is likely. + const preload = () => { + loadIndex(indexUrl).catch(() => undefined); + }; + + return ( +
+ + {isOpen && createPortal(, document.body)} +
+ ); +} diff --git a/src/plugins/search/theme/SearchBar/styles.module.css b/src/plugins/search/theme/SearchBar/styles.module.css new file mode 100644 index 00000000..dd379a86 --- /dev/null +++ b/src/plugins/search/theme/SearchBar/styles.module.css @@ -0,0 +1,412 @@ +/* Navbar button */ + +.trigger { + display: flex; + align-items: center; + gap: 0.5rem; + width: 13.5rem; + height: 2.25rem; + padding: 0 0.6rem 0 0.75rem; + border: 1px solid var(--ifm-color-emphasis-300); + border-radius: 0.75rem; + background: var(--ifm-color-emphasis-100); + color: var(--ifm-color-emphasis-700); + font: inherit; + font-size: 0.9rem; + cursor: pointer; + transition: border-color 0.15s, background-color 0.15s; +} + +.trigger:hover, +.trigger:focus-visible { + border-color: var(--ifm-color-primary); + background: var(--ifm-background-color); + outline: none; +} + +.triggerLabel { + flex: 1; + text-align: left; +} + +.triggerShortcut, +.footer kbd, +.close { + padding: 0.1rem 0.4rem; + border: 1px solid var(--ifm-color-emphasis-300); + border-radius: 0.35rem; + background: var(--ifm-background-color); + box-shadow: none; + color: var(--ifm-color-emphasis-700); + font-family: var(--ifm-font-family-base); + font-size: 0.7rem; + font-weight: 600; + line-height: 1.4; +} + +@media (max-width: 996px) { + .trigger { + width: 2.25rem; + justify-content: center; + padding: 0; + } + + .triggerLabel, + .triggerShortcut { + display: none; + } +} + +/* Dialog */ + +.overlay { + --search-border: var(--ifm-color-emphasis-300); + --search-muted: var(--ifm-color-emphasis-600); + --search-surface: var(--ifm-background-surface-color); + --search-hover: rgba(76, 76, 255, 0.08); + --search-mark: rgba(76, 76, 255, 0.16); + + position: fixed; + inset: 0; + z-index: 1000; + display: flex; + align-items: flex-start; + justify-content: center; + padding: 10vh 1rem 1rem; + background: rgba(16, 18, 40, 0.55); + backdrop-filter: blur(3px); +} + +html[data-theme='dark'] .overlay { + --search-hover: rgba(76, 140, 255, 0.14); + --search-mark: rgba(76, 140, 255, 0.28); +} + +.dialog { + display: flex; + flex-direction: column; + width: 100%; + max-width: 46rem; + max-height: min(80vh, 44rem); + overflow: hidden; + border: 1px solid var(--search-border); + border-radius: 0.9rem; + background: var(--search-surface); + box-shadow: 0 24px 60px rgba(0, 0, 0, 0.3); + color: var(--ifm-font-color-base); +} + +.inputRow { + display: flex; + flex-shrink: 0; + align-items: center; + gap: 0.75rem; + padding: 0.85rem 1rem; + border-bottom: 1px solid var(--search-border); + color: var(--ifm-color-primary); +} + +.input { + flex: 1; + min-width: 0; + border: 0; + outline: 0; + background: transparent; + color: var(--ifm-font-color-base); + font: inherit; + font-size: 1.05rem; + appearance: none; +} + +.input::placeholder { + color: var(--search-muted); +} + +.input::-webkit-search-cancel-button { + display: none; +} + +.close { + cursor: pointer; +} + +.close:hover { + border-color: var(--ifm-color-primary); + color: var(--ifm-color-primary); +} + +.filters { + display: flex; + flex-shrink: 0; + gap: 0.4rem; + padding: 0.6rem 1rem; + overflow-x: auto; + border-bottom: 1px solid var(--search-border); +} + +.filter { + display: inline-flex; + align-items: center; + gap: 0.4rem; + padding: 0.2rem 0.7rem; + border: 1px solid var(--search-border); + border-radius: 999px; + background: transparent; + color: var(--ifm-color-emphasis-800); + font: inherit; + font-size: 0.8rem; + font-weight: 500; + white-space: nowrap; + cursor: pointer; +} + +.filter:hover { + border-color: var(--ifm-color-primary); +} + +.filterActive { + border-color: var(--ifm-color-primary); + background: var(--ifm-color-primary); + color: #fff; +} + +.count { + font-size: 0.72rem; + font-variant-numeric: tabular-nums; + opacity: 0.7; +} + +.results { + flex: 1; + min-height: 0; + padding: 0.5rem; + overflow-y: auto; + overscroll-behavior: contain; +} + +.message { + margin: 0; + padding: 2rem 1rem; + color: var(--search-muted); + font-size: 0.95rem; + text-align: center; +} + +.message p { + margin: 0 0 1rem; +} + +.suggestions { + display: flex; + flex-wrap: wrap; + justify-content: center; + gap: 0.5rem; +} + +.suggestion { + padding: 0.25rem 0.75rem; + border: 1px solid var(--search-border); + border-radius: 999px; + background: transparent; + color: var(--ifm-font-color-base); + font: inherit; + font-size: 0.85rem; + cursor: pointer; +} + +.suggestion:hover { + border-color: var(--ifm-color-primary); + color: var(--ifm-color-primary); +} + +/* Results */ + +.group + .group { + margin-top: 0.35rem; + padding-top: 0.35rem; + border-top: 1px solid var(--ifm-color-emphasis-200); +} + +.row { + display: block; + padding: 0.5rem 0.75rem; + border-radius: 0.5rem; + color: inherit; + text-decoration: none; +} + +.row:hover { + color: inherit; + text-decoration: none; +} + +.selected { + background: var(--search-hover); + box-shadow: inset 3px 0 0 var(--ifm-color-primary); +} + +.sectionRow { + margin-left: 1rem; +} + +.rowTitle { + display: flex; + align-items: baseline; + flex-wrap: wrap; + gap: 0.15rem 0.5rem; + min-width: 0; +} + +.pageTitle { + font-size: 0.98rem; + font-weight: 600; +} + +.rowHeading { + font-size: 0.9rem; + font-weight: 500; +} + +.hash { + color: var(--ifm-color-emphasis-500); + font-weight: 600; +} + +.breadcrumbs { + color: var(--search-muted); + font-size: 0.78rem; +} + +.kind { + align-self: center; + padding: 0.05rem 0.4rem; + border-radius: 0.3rem; + background: var(--ifm-color-emphasis-200); + color: var(--ifm-color-emphasis-800); + font-size: 0.66rem; + font-weight: 700; + letter-spacing: 0.04em; + text-transform: uppercase; +} + +.kind-api { + background: rgba(0, 165, 255, 0.16); + color: #0a6fb0; +} + +.kind-changelog { + background: rgba(240, 160, 0, 0.18); + color: #94610a; +} + +html[data-theme='dark'] .kind-api { + color: #6cc7ff; +} + +html[data-theme='dark'] .kind-changelog { + color: #f5c264; +} + +.excerpt { + display: -webkit-box; + margin-top: 0.15rem; + overflow: hidden; + color: var(--ifm-color-emphasis-700); + font-size: 0.84rem; + line-height: 1.45; + -webkit-box-orient: vertical; + -webkit-line-clamp: 2; +} + +.excerptCode { + font-family: var(--ifm-font-family-monospace); + font-size: 0.78rem; +} + +.row mark { + padding: 0 0.1em; + border-radius: 0.2rem; + background: var(--search-mark); + color: var(--ifm-color-primary-darkest); + font-weight: 600; +} + +html[data-theme='dark'] .row mark { + color: var(--ifm-color-primary-lightest); +} + +.endpoint { + overflow: hidden; + color: var(--search-muted); + font-family: var(--ifm-font-family-monospace); + font-size: 0.76rem; + text-overflow: ellipsis; + white-space: nowrap; +} + +.method { + margin-right: 0.4rem; + font-weight: 700; +} + +.methodGET { + color: #1a8a4a; +} + +.methodPOST { + color: #2d6fdb; +} + +.methodPUT, +.methodPATCH { + color: #b7791f; +} + +.methodDELETE { + color: #d64545; +} + +.deprecated { + padding: 0 0.35rem; + border-radius: 0.3rem; + background: var(--ifm-color-emphasis-200); + color: var(--ifm-color-emphasis-700); + font-size: 0.68rem; + font-weight: 600; +} + +.footer { + display: flex; + flex-shrink: 0; + gap: 1.25rem; + padding: 0.55rem 1rem; + border-top: 1px solid var(--search-border); + color: var(--search-muted); + font-size: 0.75rem; +} + +.footer kbd { + margin-right: 0.25rem; +} + +@media (max-width: 600px) { + .overlay { + padding: 0; + } + + .dialog { + max-width: none; + height: 100%; + max-height: none; + border: 0; + border-radius: 0; + } + + .footer { + display: none; + } + + .sectionRow { + margin-left: 0.5rem; + } +}