refactor: migrate from shiki to shikiji (#2520)

This commit is contained in:
ocavue 2023-12-21 02:54:40 +08:00 committed by GitHub
parent e63473a5f8
commit 74138a9a58
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 124 additions and 223 deletions

View file

@ -11,7 +11,6 @@ dist
.netlify/
.eslintcache
public/shiki
public/emojis
*~

1
.gitignore vendored
View file

@ -11,7 +11,6 @@ dist
.eslintcache
elk-translation-status.json
public/shiki
public/emojis
*~

View file

@ -151,7 +151,7 @@ You can consult the [PWA documentation](https://docs.elk.zone/pwa) to learn more
- [UnoCSS](https://uno.antfu.me/) - The instant on-demand atomic CSS engine
- [Iconify](https://github.com/iconify/icon-sets#iconify-icon-sets-in-json-format) - Iconify icon sets in JSON format
- [Masto.js](https://neet.github.io/masto.js) - Mastodon API client in TypeScript
- [shiki](https://shiki.matsu.io/) - A beautiful Syntax Highlighter
- [shikiji](https://shikiji.netlify.app/) - A beautiful and powerful syntax highlighter
- [vite-plugin-pwa](https://github.com/vite-pwa/vite-plugin-pwa) - Prompt for update, Web Push Notifications and Web Share Target API
## 👨‍💻 Contributors

View file

@ -1,16 +1,15 @@
import type { Highlighter, Lang } from 'shiki-es'
import { type Highlighter, type BuiltinLanguage as Lang } from 'shikiji'
const shiki = ref<Highlighter>()
const highlighter = ref<Highlighter>()
const registeredLang = ref(new Map<string, boolean>())
let shikiImport: Promise<void> | undefined
let shikijiImport: Promise<void> | undefined
export function useHighlighter(lang: Lang) {
if (!shikiImport) {
shikiImport = import('shiki-es')
.then(async (r) => {
r.setCDN('/shiki/')
shiki.value = await r.getHighlighter({
if (!shikijiImport) {
shikijiImport = import('shikiji')
.then(async ({ getHighlighter }) => {
highlighter.value = await getHighlighter({
themes: [
'vitesse-dark',
'vitesse-light',
@ -24,27 +23,27 @@ export function useHighlighter(lang: Lang) {
})
}
if (!shiki.value)
if (!highlighter.value)
return undefined
if (!registeredLang.value.get(lang)) {
shiki.value.loadLanguage(lang)
highlighter.value.loadLanguage(lang)
.then(() => {
registeredLang.value.set(lang, true)
})
.catch(() => {
const fallbackLang = 'md'
shiki.value?.loadLanguage(fallbackLang).then(() => {
highlighter.value?.loadLanguage(fallbackLang).then(() => {
registeredLang.value.set(fallbackLang, true)
})
})
return undefined
}
return shiki.value
return highlighter.value
}
export function useShikiTheme() {
function useShikijiTheme() {
return useColorMode().value === 'dark' ? 'vitesse-dark' : 'vitesse-light'
}
@ -61,16 +60,12 @@ function escapeHtml(text: string) {
}
export function highlightCode(code: string, lang: Lang) {
const shiki = useHighlighter(lang)
if (!shiki)
const highlighter = useHighlighter(lang)
if (!highlighter)
return escapeHtml(code)
return shiki.codeToHtml(code, {
return highlighter.codeToHtml(code, {
lang,
theme: useShikiTheme(),
theme: useShikijiTheme(),
})
}
export function useShiki() {
return shiki
}

View file

@ -14,7 +14,7 @@ import { Plugin } from 'prosemirror-state'
import type { Ref } from 'vue'
import { TiptapEmojiSuggestion, TiptapHashtagSuggestion, TiptapMentionSuggestion } from './tiptap/suggestion'
import { TiptapPluginCodeBlockShiki } from './tiptap/shiki'
import { TiptapPluginCodeBlockShikiji } from './tiptap/shikiji'
import { TiptapPluginCustomEmoji } from './tiptap/custom-emoji'
import { TiptapPluginEmoji } from './tiptap/emoji'
@ -70,7 +70,7 @@ export function useTiptap(options: UseTiptapOptions) {
Placeholder.configure({
placeholder: () => placeholder.value!,
}),
TiptapPluginCodeBlockShiki,
TiptapPluginCodeBlockShikiji,
History.configure({
depth: 10,
}),

View file

@ -1,129 +0,0 @@
import type { CodeBlockOptions } from '@tiptap/extension-code-block'
import CodeBlock from '@tiptap/extension-code-block'
import { VueNodeViewRenderer } from '@tiptap/vue-3'
import { findChildren } from '@tiptap/core'
import type { Node as ProsemirrorNode } from 'prosemirror-model'
import { Plugin, PluginKey } from 'prosemirror-state'
import { Decoration, DecorationSet } from 'prosemirror-view'
import TiptapCodeBlock from '~/components/tiptap/TiptapCodeBlock.vue'
export interface CodeBlockShikiOptions extends CodeBlockOptions {
defaultLanguage: string | null | undefined
}
export const TiptapPluginCodeBlockShiki = CodeBlock.extend<CodeBlockShikiOptions>({
addOptions() {
return {
...this.parent?.(),
defaultLanguage: null,
}
},
addProseMirrorPlugins() {
return [
...this.parent?.() || [],
ProseMirrorShikiPlugin({
name: this.name,
}),
]
},
addNodeView() {
return VueNodeViewRenderer(TiptapCodeBlock)
},
})
function getDecorations({
doc,
name,
}: { doc: ProsemirrorNode; name: string }) {
const decorations: Decoration[] = []
findChildren(doc, node => node.type.name === name)
.forEach((block) => {
let from = block.pos + 1
const language = block.node.attrs.language
const shiki = useHighlighter(language)
if (!shiki)
return
const lines = shiki.codeToThemedTokens(block.node.textContent, language, useShikiTheme())
lines.forEach((line) => {
line.forEach((token) => {
const decoration = Decoration.inline(from, from + token.content.length, {
style: `color: ${token.color}`,
})
decorations.push(decoration)
from += token.content.length
})
from += 1
})
})
return DecorationSet.create(doc, decorations)
}
function ProseMirrorShikiPlugin({ name }: { name: string }) {
const plugin: Plugin<any> = new Plugin({
key: new PluginKey('shiki'),
state: {
init: (_, { doc }) => getDecorations({
doc,
name,
}),
apply: (transaction, decorationSet, oldState, newState) => {
const oldNodeName = oldState.selection.$head.parent.type.name
const newNodeName = newState.selection.$head.parent.type.name
const oldNodes = findChildren(oldState.doc, node => node.type.name === name)
const newNodes = findChildren(newState.doc, node => node.type.name === name)
if (
transaction.docChanged
// Apply decorations if:
&& (
// selection includes named node,
[oldNodeName, newNodeName].includes(name)
// OR transaction adds/removes named node,
|| newNodes.length !== oldNodes.length
// OR transaction has changes that completely encapsulte a node
// (for example, a transaction that affects the entire document).
// Such transactions can happen during collab syncing via y-prosemirror, for example.
|| transaction.steps.some((step) => {
// @ts-expect-error cast
return step.from !== undefined
// @ts-expect-error cast
&& step.to !== undefined
&& oldNodes.some((node) => {
// @ts-expect-error cast
return node.pos >= step.from
// @ts-expect-error cast
&& node.pos + node.node.nodeSize <= step.to
})
})
)
) {
return getDecorations({
doc: transaction.doc,
name,
})
}
return decorationSet.map(transaction.mapping, transaction.doc)
},
},
props: {
decorations(state) {
return plugin.getState(state)
},
},
})
return plugin
}

View file

@ -0,0 +1,20 @@
import { type Parser, createParser } from 'prosemirror-highlight/shikiji'
import type { BuiltinLanguage } from 'shikiji/langs'
let parser: Parser | undefined
export const shikijiParser: Parser = (options) => {
const lang = options.language ?? 'text'
// Register the language if it's not yet registered
const highlighter = useHighlighter(lang as BuiltinLanguage)
// If the language is not loaded, we return an empty set of decorations
if (!highlighter)
return []
if (!parser)
parser = createParser(highlighter)
return parser(options)
}

View file

@ -0,0 +1,25 @@
import CodeBlock from '@tiptap/extension-code-block'
import { VueNodeViewRenderer } from '@tiptap/vue-3'
import { createHighlightPlugin } from 'prosemirror-highlight'
import { shikijiParser } from './shikiji-parser'
import TiptapCodeBlock from '~/components/tiptap/TiptapCodeBlock.vue'
export const TiptapPluginCodeBlockShikiji = CodeBlock.extend({
addOptions() {
return {
...this.parent?.(),
defaultLanguage: null,
}
},
addProseMirrorPlugins() {
return [
createHighlightPlugin({ parser: shikijiParser, nodeTypes: ['codeBlock'] }),
]
},
addNodeView() {
return VueNodeViewRenderer(TiptapCodeBlock)
},
})

View file

@ -14,7 +14,7 @@ export const pwa: VitePWANuxtOptions = {
manifest: false,
injectManifest: {
globPatterns: ['**/*.{js,json,css,html,txt,svg,png,ico,webp,woff,woff2,ttf,eot,otf,wasm}'],
globIgnores: ['emojis/**', 'shiki/**', 'manifest**.webmanifest'],
globIgnores: ['emojis/**', 'manifest**.webmanifest'],
},
devOptions: {
enabled: process.env.VITE_DEV_PWA === 'true',

View file

@ -49,5 +49,5 @@ nr test
- [UnoCSS](https://uno.antfu.me/) - The instant on-demand atomic CSS engine
- [Iconify](https://github.com/iconify/icon-sets#iconify-icon-sets-in-json-format) - Iconify icon sets in JSON format
- [Masto.js](https://neet.github.io/masto.js) - Mastodon API client in TypeScript
- [shiki](https://shiki.matsu.io/) - A beautiful Syntax Highlighter
- [shikiji](https://shikiji.netlify.app/) - A beautiful and powerful syntax highlighter
- [vite-plugin-pwa](https://github.com/vite-pwa/vite-plugin-pwa) - Prompt for update, Web Push Notifications and Web Share Target API

View file

@ -2,7 +2,7 @@ import proxy from 'unenv/runtime/mock/proxy'
export const Plugin = proxy
export const PluginKey = proxy
export const Decoration = proxy
export const DecorationSet = proxy
export const createParser = proxy
export const createHighlightPlugin = proxy
export { proxy as default }

View file

@ -157,11 +157,6 @@ export default defineNuxtConfig({
maxAge: 24 * 60 * 60 * 365, // 1 year (versioned)
baseURL: '/fonts',
},
{
dir: '~/public/shiki',
maxAge: 24 * 60 * 60 * 365, // 1 year, matching service worker
baseURL: '/shiki',
},
],
},
sourcemap: isDevelopment,
@ -179,7 +174,7 @@ export default defineNuxtConfig({
const alias = config.resolve!.alias as Record<string, string>
for (const dep of ['eventemitter3', 'isomorphic-ws'])
alias[dep] = resolve('./mocks/class')
for (const dep of ['shiki-es', 'fuse.js'])
for (const dep of ['fuse.js'])
alias[dep] = 'unenv/runtime/mock/proxy'
const resolver = createResolver(import.meta.url)

View file

@ -83,9 +83,9 @@
"page-lifecycle": "^0.1.2",
"pinia": "^2.1.4",
"postcss-nested": "^6.0.1",
"prosemirror-highlight": "^0.3.3",
"rollup-plugin-node-polyfills": "^0.2.1",
"shiki": "^0.14.3",
"shiki-es": "^0.2.0",
"shikiji": "^0.9.9",
"simple-git": "^3.19.1",
"slimeform": "^0.9.1",
"stale-dep": "^0.7.0",

View file

@ -176,15 +176,15 @@ importers:
postcss-nested:
specifier: ^6.0.1
version: 6.0.1(postcss@8.4.32)
prosemirror-highlight:
specifier: ^0.3.3
version: 0.3.3(prosemirror-model@1.19.2)(prosemirror-state@1.4.3)(prosemirror-view@1.31.5)(shikiji@0.9.9)
rollup-plugin-node-polyfills:
specifier: ^0.2.1
version: 0.2.1
shiki:
specifier: ^0.14.3
version: 0.14.3
shiki-es:
specifier: ^0.2.0
version: 0.2.0
shikiji:
specifier: ^0.9.9
version: 0.9.9
simple-git:
specifier: ^3.19.1
version: 3.21.0
@ -6318,10 +6318,6 @@ packages:
resolution: {integrity: sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==}
engines: {node: '>=12'}
/ansi-sequence-parser@1.1.0:
resolution: {integrity: sha512-lEm8mt52to2fT8GhciPCGeCXACSz2UwIN4X2e2LJSnZ5uAbn2/dsYdOmUXq0AtWS5cpAupysIneExOgH0Vd2TQ==}
dev: false
/ansi-styles@3.2.1:
resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==}
engines: {node: '>=4'}
@ -12095,6 +12091,47 @@ packages:
prosemirror-view: 1.31.5
dev: false
/prosemirror-highlight@0.3.3(prosemirror-model@1.19.2)(prosemirror-state@1.4.3)(prosemirror-view@1.31.5)(shikiji@0.9.9):
resolution: {integrity: sha512-tOGyPvmRKZ49ubzKmFIwiwS7CNXlU9g/D4zZLaHGzXLVNVnBrmbDOajZ4eP0lylOAWPxZN+vrFZ9DwrtyikuoA==}
peerDependencies:
'@types/hast': ^3.0.0
highlight.js: ^11.9.0
lowlight: ^3.1.0
prosemirror-model: ^1.19.3
prosemirror-state: ^1.4.3
prosemirror-transform: ^1.8.0
prosemirror-view: ^1.32.4
refractor: ^4.8.1
shiki: ^0.14.6
shikiji: ^0.8.0 || ^0.9.0
peerDependenciesMeta:
'@types/hast':
optional: true
highlight.js:
optional: true
lowlight:
optional: true
prosemirror-model:
optional: true
prosemirror-state:
optional: true
prosemirror-transform:
optional: true
prosemirror-view:
optional: true
refractor:
optional: true
shiki:
optional: true
shikiji:
optional: true
dependencies:
prosemirror-model: 1.19.2
prosemirror-state: 1.4.3
prosemirror-view: 1.31.5
shikiji: 0.9.9
dev: false
/prosemirror-history@1.3.2:
resolution: {integrity: sha512-/zm0XoU/N/+u7i5zepjmZAEnpvjDtzoPWW6VmKptcAnPadN/SStsBjMImdCEbb3seiNTpveziPTIrXQbHLtU1g==}
dependencies:
@ -12887,17 +12924,14 @@ packages:
resolution: {integrity: sha512-e+/aueHx0YeIEut6RXC6K8gSf0PykwZiHD7q7AHtpTW8Kd8TpFUIWqTwhAnrGjOyOMyrwv+syr5WPagMpDpVYQ==}
dev: true
/shiki-es@0.2.0:
resolution: {integrity: sha512-RbRMD+IuJJseSZljDdne9ThrUYrwBwJR04FvN4VXpfsU3MNID5VJGHLAD5je/HGThCyEKNgH+nEkSFEWKD7C3Q==}
/shikiji-core@0.9.9:
resolution: {integrity: sha512-qu5Qq7Co6JIMY312J9Ek6WYjXieeyJT/fIqmkcjF4MdnMNlUnhSqPo8/42g5UdPgdyTCwijS7Nhg8DfLSLodkg==}
dev: false
/shiki@0.14.3:
resolution: {integrity: sha512-U3S/a+b0KS+UkTyMjoNojvTgrBHjgp7L6ovhFVZsXmBGnVdQ4K4U9oK0z63w538S91ATngv1vXigHCSWOwnr+g==}
/shikiji@0.9.9:
resolution: {integrity: sha512-/S3unr/0mZTstNOuAmNDEufeimtqeQb8lXvPMLsYfDvqyfmG6334bO2xmDzD0kfxH2y8gnFgSWAJpdEzksmYXg==}
dependencies:
ansi-sequence-parser: 1.1.0
jsonc-parser: 3.2.0
vscode-oniguruma: 1.7.0
vscode-textmate: 8.0.0
shikiji-core: 0.9.9
dev: false
/side-channel@1.0.4:
@ -14708,14 +14742,6 @@ packages:
dependencies:
vscode-languageserver-protocol: 3.16.0
/vscode-oniguruma@1.7.0:
resolution: {integrity: sha512-L9WMGRfrjOhgHSdOYgCt/yRMsXzLDJSL7BPrOZt73gU0iWO4mpqzqQzOz5srxqTvMBaR0XZTSrVWo4j55Rc6cA==}
dev: false
/vscode-textmate@8.0.0:
resolution: {integrity: sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==}
dev: false
/vscode-uri@3.0.7:
resolution: {integrity: sha512-eOpPHogvorZRobNqJGhapa0JdwaxpjVvyBp0QIUMRMSf8ZAlqOdEquKuRmw9Qwu0qXtJIWqFtMkmvJjUZmMjVA==}

View file

@ -5,12 +5,6 @@ import { colorsMap } from './generate-themes'
const dereference = process.platform === 'win32' ? true : undefined
await fs.copy('node_modules/shiki-es/dist/assets', 'public/shiki/', {
dereference,
filter: src => src === 'node_modules/shiki/' || src.includes('languages') || src.includes('dist'),
})
await fs.copy('node_modules/theme-vitesse/themes', 'public/shiki/themes', { dereference })
await fs.copy('node_modules/theme-vitesse/themes', 'node_modules/shiki/themes', { overwrite: true, dereference })
await fs.copy(`node_modules/${iconifyEmojiPackage}/icons`, `public/emojis/${emojiPrefix}`, { overwrite: true, dereference })
await fs.writeJSON('constants/themes.json', colorsMap, { spaces: 2, EOL: '\n' })

View file

@ -39,9 +39,7 @@ if (import.meta.env.PROD) {
/^\/oauth\//,
/^\/signin\//,
/^\/web-share-target\//,
// exclude shiki: has its own cache
/^\/shiki\//,
// exclude shiki: has its own cache
// exclude emoji: has its own cache
/^\/emojis\//,
// exclude sw: if the user navigates to it, fallback to index.html
/^\/sw.js$/,
@ -65,19 +63,6 @@ if (import.meta.env.PROD) {
],
}),
)
// include shiki cache
registerRoute(
({ sameOrigin, url }) =>
sameOrigin && url.pathname.startsWith('/shiki/'),
new StaleWhileRevalidate({
cacheName: 'elk-shiki',
plugins: [
new CacheableResponsePlugin({ statuses: [200] }),
// 365 days max
new ExpirationPlugin({ purgeOnQuotaError: true, maxAgeSeconds: 60 * 60 * 24 * 365 }),
],
}),
)
// include emoji icons
registerRoute(
({ sameOrigin, request, url }) =>

View file

@ -279,14 +279,6 @@ vi.mock('vue-router', async () => {
}
})
vi.mock('shiki-es', async (importOriginal) => {
const mod = await importOriginal()
return {
...(mod as any),
setCDN() {},
}
})
mockComponent('ContentMentionGroup', {
setup(props, { slots }) {
return () => h('mention-group', null, { default: () => slots?.default?.() })