refactor: migrate from shiki to shikiji (#2520)

.dockerignore

@@ -11,7 +11,6 @@ dist
 .netlify/
 .eslintcache
 
-public/shiki
 public/emojis
 
 *~

.gitignore

@@ -11,7 +11,6 @@ dist
 .eslintcache
 elk-translation-status.json
 
-public/shiki
 public/emojis
 
 *~

README.md

@@ -151,7 +151,7 @@ You can consult the [PWA documentation](https://docs.elk.zone/pwa) to learn more
 - [UnoCSS](https://uno.antfu.me/) - The instant on-demand atomic CSS engine
 - [Iconify](https://github.com/iconify/icon-sets#iconify-icon-sets-in-json-format) - Iconify icon sets in JSON format
 - [Masto.js](https://neet.github.io/masto.js) - Mastodon API client in TypeScript
-- [shiki](https://shiki.matsu.io/) - A beautiful Syntax Highlighter
+- [shikiji](https://shikiji.netlify.app/) - A beautiful and powerful syntax highlighter
 - [vite-plugin-pwa](https://github.com/vite-pwa/vite-plugin-pwa) - Prompt for update, Web Push Notifications and Web Share Target API
 
 ## 👨‍💻 Contributors

composables/shikiji.ts

@@ -1,16 +1,15 @@
-import type { Highlighter, Lang } from 'shiki-es'
+import { type Highlighter, type BuiltinLanguage as Lang } from 'shikiji'
 
-const shiki = ref<Highlighter>()
+const highlighter = ref<Highlighter>()
 
 const registeredLang = ref(new Map<string, boolean>())
-let shikiImport: Promise<void> | undefined
+let shikijiImport: Promise<void> | undefined
 
 export function useHighlighter(lang: Lang) {
-  if (!shikiImport) {
+  if (!shikijiImport) {
-    shikiImport = import('shiki-es')
+    shikijiImport = import('shikiji')
-      .then(async (r) => {
+      .then(async ({ getHighlighter }) => {
-        r.setCDN('/shiki/')
+        highlighter.value = await getHighlighter({
-        shiki.value = await r.getHighlighter({
           themes: [
             'vitesse-dark',
             'vitesse-light',
@@ -24,27 +23,27 @@ export function useHighlighter(lang: Lang) {
       })
   }
 
-  if (!shiki.value)
+  if (!highlighter.value)
     return undefined
 
   if (!registeredLang.value.get(lang)) {
-    shiki.value.loadLanguage(lang)
+    highlighter.value.loadLanguage(lang)
       .then(() => {
         registeredLang.value.set(lang, true)
       })
       .catch(() => {
         const fallbackLang = 'md'
-        shiki.value?.loadLanguage(fallbackLang).then(() => {
+        highlighter.value?.loadLanguage(fallbackLang).then(() => {
           registeredLang.value.set(fallbackLang, true)
         })
       })
     return undefined
   }
 
-  return shiki.value
+  return highlighter.value
 }
 
-export function useShikiTheme() {
+function useShikijiTheme() {
   return useColorMode().value === 'dark' ? 'vitesse-dark' : 'vitesse-light'
 }
 
@@ -61,16 +60,12 @@ function escapeHtml(text: string) {
 }
 
 export function highlightCode(code: string, lang: Lang) {
-  const shiki = useHighlighter(lang)
+  const highlighter = useHighlighter(lang)
-  if (!shiki)
+  if (!highlighter)
     return escapeHtml(code)
 
-  return shiki.codeToHtml(code, {
+  return highlighter.codeToHtml(code, {
     lang,
-    theme: useShikiTheme(),
+    theme: useShikijiTheme(),
   })
 }
-
-export function useShiki() {
-  return shiki
-}

composables/tiptap.ts

@@ -14,7 +14,7 @@ import { Plugin } from 'prosemirror-state'
 
 import type { Ref } from 'vue'
 import { TiptapEmojiSuggestion, TiptapHashtagSuggestion, TiptapMentionSuggestion } from './tiptap/suggestion'
-import { TiptapPluginCodeBlockShiki } from './tiptap/shiki'
+import { TiptapPluginCodeBlockShikiji } from './tiptap/shikiji'
 import { TiptapPluginCustomEmoji } from './tiptap/custom-emoji'
 import { TiptapPluginEmoji } from './tiptap/emoji'
 
@@ -70,7 +70,7 @@ export function useTiptap(options: UseTiptapOptions) {
       Placeholder.configure({
         placeholder: () => placeholder.value!,
       }),
-      TiptapPluginCodeBlockShiki,
+      TiptapPluginCodeBlockShikiji,
       History.configure({
         depth: 10,
       }),

composables/tiptap/shiki.ts

@@ -1,129 +0,0 @@
-import type { CodeBlockOptions } from '@tiptap/extension-code-block'
-import CodeBlock from '@tiptap/extension-code-block'
-import { VueNodeViewRenderer } from '@tiptap/vue-3'
-
-import { findChildren } from '@tiptap/core'
-import type { Node as ProsemirrorNode } from 'prosemirror-model'
-import { Plugin, PluginKey } from 'prosemirror-state'
-import { Decoration, DecorationSet } from 'prosemirror-view'
-import TiptapCodeBlock from '~/components/tiptap/TiptapCodeBlock.vue'
-
-export interface CodeBlockShikiOptions extends CodeBlockOptions {
-  defaultLanguage: string | null | undefined
-}
-
-export const TiptapPluginCodeBlockShiki = CodeBlock.extend<CodeBlockShikiOptions>({
-  addOptions() {
-    return {
-      ...this.parent?.(),
-      defaultLanguage: null,
-    }
-  },
-
-  addProseMirrorPlugins() {
-    return [
-      ...this.parent?.() || [],
-      ProseMirrorShikiPlugin({
-        name: this.name,
-      }),
-    ]
-  },
-
-  addNodeView() {
-    return VueNodeViewRenderer(TiptapCodeBlock)
-  },
-})
-
-function getDecorations({
-  doc,
-  name,
-}: { doc: ProsemirrorNode; name: string }) {
-  const decorations: Decoration[] = []
-
-  findChildren(doc, node => node.type.name === name)
-    .forEach((block) => {
-      let from = block.pos + 1
-      const language = block.node.attrs.language
-
-      const shiki = useHighlighter(language)
-
-      if (!shiki)
-        return
-
-      const lines = shiki.codeToThemedTokens(block.node.textContent, language, useShikiTheme())
-
-      lines.forEach((line) => {
-        line.forEach((token) => {
-          const decoration = Decoration.inline(from, from + token.content.length, {
-            style: `color: ${token.color}`,
-          })
-
-          decorations.push(decoration)
-          from += token.content.length
-        })
-        from += 1
-      })
-    })
-
-  return DecorationSet.create(doc, decorations)
-}
-
-function ProseMirrorShikiPlugin({ name }: { name: string }) {
-  const plugin: Plugin<any> = new Plugin({
-    key: new PluginKey('shiki'),
-
-    state: {
-      init: (_, { doc }) => getDecorations({
-        doc,
-        name,
-      }),
-      apply: (transaction, decorationSet, oldState, newState) => {
-        const oldNodeName = oldState.selection.$head.parent.type.name
-        const newNodeName = newState.selection.$head.parent.type.name
-        const oldNodes = findChildren(oldState.doc, node => node.type.name === name)
-        const newNodes = findChildren(newState.doc, node => node.type.name === name)
-
-        if (
-          transaction.docChanged
-          // Apply decorations if:
-          && (
-            // selection includes named node,
-            [oldNodeName, newNodeName].includes(name)
-            // OR transaction adds/removes named node,
-            || newNodes.length !== oldNodes.length
-            // OR transaction has changes that completely encapsulte a node
-            // (for example, a transaction that affects the entire document).
-            // Such transactions can happen during collab syncing via y-prosemirror, for example.
-            || transaction.steps.some((step) => {
-              // @ts-expect-error cast
-              return step.from !== undefined
-                // @ts-expect-error cast
-                && step.to !== undefined
-                && oldNodes.some((node) => {
-                  // @ts-expect-error cast
-                  return node.pos >= step.from
-                    // @ts-expect-error cast
-                    && node.pos + node.node.nodeSize <= step.to
-                })
-            })
-          )
-        ) {
-          return getDecorations({
-            doc: transaction.doc,
-            name,
-          })
-        }
-
-        return decorationSet.map(transaction.mapping, transaction.doc)
-      },
-    },
-
-    props: {
-      decorations(state) {
-        return plugin.getState(state)
-      },
-    },
-  })
-
-  return plugin
-}

composables/tiptap/shikiji-parser.ts

@@ -0,0 +1,20 @@
+import { type Parser, createParser } from 'prosemirror-highlight/shikiji'
+import type { BuiltinLanguage } from 'shikiji/langs'
+
+let parser: Parser | undefined
+
+export const shikijiParser: Parser = (options) => {
+  const lang = options.language ?? 'text'
+
+  // Register the language if it's not yet registered
+  const highlighter = useHighlighter(lang as BuiltinLanguage)
+
+  // If the language is not loaded, we return an empty set of decorations
+  if (!highlighter)
+    return []
+
+  if (!parser)
+    parser = createParser(highlighter)
+
+  return parser(options)
+}

composables/tiptap/shikiji.ts

@@ -0,0 +1,25 @@
+import CodeBlock from '@tiptap/extension-code-block'
+import { VueNodeViewRenderer } from '@tiptap/vue-3'
+
+import { createHighlightPlugin } from 'prosemirror-highlight'
+import { shikijiParser } from './shikiji-parser'
+import TiptapCodeBlock from '~/components/tiptap/TiptapCodeBlock.vue'
+
+export const TiptapPluginCodeBlockShikiji = CodeBlock.extend({
+  addOptions() {
+    return {
+      ...this.parent?.(),
+      defaultLanguage: null,
+    }
+  },
+
+  addProseMirrorPlugins() {
+    return [
+      createHighlightPlugin({ parser: shikijiParser, nodeTypes: ['codeBlock'] }),
+    ]
+  },
+
+  addNodeView() {
+    return VueNodeViewRenderer(TiptapCodeBlock)
+  },
+})

config/pwa.ts

@@ -14,7 +14,7 @@ export const pwa: VitePWANuxtOptions = {
   manifest: false,
   injectManifest: {
     globPatterns: ['**/*.{js,json,css,html,txt,svg,png,ico,webp,woff,woff2,ttf,eot,otf,wasm}'],
-    globIgnores: ['emojis/**', 'shiki/**', 'manifest**.webmanifest'],
+    globIgnores: ['emojis/**', 'manifest**.webmanifest'],
   },
   devOptions: {
     enabled: process.env.VITE_DEV_PWA === 'true',

docs/content/1.guide/3.contributing.md

@@ -49,5 +49,5 @@ nr test
 - [UnoCSS](https://uno.antfu.me/) - The instant on-demand atomic CSS engine
 - [Iconify](https://github.com/iconify/icon-sets#iconify-icon-sets-in-json-format) - Iconify icon sets in JSON format
 - [Masto.js](https://neet.github.io/masto.js) - Mastodon API client in TypeScript
-- [shiki](https://shiki.matsu.io/) - A beautiful Syntax Highlighter
+- [shikiji](https://shikiji.netlify.app/) - A beautiful and powerful syntax highlighter
 - [vite-plugin-pwa](https://github.com/vite-pwa/vite-plugin-pwa) - Prompt for update, Web Push Notifications and Web Share Target API

mocks/prosemirror.ts

@@ -2,7 +2,7 @@ import proxy from 'unenv/runtime/mock/proxy'
 
 export const Plugin = proxy
 export const PluginKey = proxy
-export const Decoration = proxy
+export const createParser = proxy
-export const DecorationSet = proxy
+export const createHighlightPlugin = proxy
 
 export { proxy as default }

nuxt.config.ts

@@ -157,11 +157,6 @@ export default defineNuxtConfig({
         maxAge: 24 * 60 * 60 * 365, // 1 year (versioned)
         baseURL: '/fonts',
       },
-      {
-        dir: '~/public/shiki',
-        maxAge: 24 * 60 * 60 * 365, // 1 year, matching service worker
-        baseURL: '/shiki',
-      },
     ],
   },
   sourcemap: isDevelopment,
@@ -179,7 +174,7 @@ export default defineNuxtConfig({
         const alias = config.resolve!.alias as Record<string, string>
         for (const dep of ['eventemitter3', 'isomorphic-ws'])
           alias[dep] = resolve('./mocks/class')
-        for (const dep of ['shiki-es', 'fuse.js'])
+        for (const dep of ['fuse.js'])
           alias[dep] = 'unenv/runtime/mock/proxy'
         const resolver = createResolver(import.meta.url)
 

package.json

@@ -83,9 +83,9 @@
     "page-lifecycle": "^0.1.2",
     "pinia": "^2.1.4",
     "postcss-nested": "^6.0.1",
+    "prosemirror-highlight": "^0.3.3",
     "rollup-plugin-node-polyfills": "^0.2.1",
-    "shiki": "^0.14.3",
+    "shikiji": "^0.9.9",
-    "shiki-es": "^0.2.0",
     "simple-git": "^3.19.1",
     "slimeform": "^0.9.1",
     "stale-dep": "^0.7.0",

pnpm-lock.yaml

@@ -176,15 +176,15 @@ importers:
       postcss-nested:
         specifier: ^6.0.1
         version: 6.0.1(postcss@8.4.32)
+      prosemirror-highlight:
+        specifier: ^0.3.3
+        version: 0.3.3(prosemirror-model@1.19.2)(prosemirror-state@1.4.3)(prosemirror-view@1.31.5)(shikiji@0.9.9)
       rollup-plugin-node-polyfills:
         specifier: ^0.2.1
         version: 0.2.1
-      shiki:
+      shikiji:
-        specifier: ^0.14.3
+        specifier: ^0.9.9
-        version: 0.14.3
+        version: 0.9.9
-      shiki-es:
-        specifier: ^0.2.0
-        version: 0.2.0
       simple-git:
         specifier: ^3.19.1
         version: 3.21.0
@@ -6318,10 +6318,6 @@ packages:
     resolution: {integrity: sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==}
     engines: {node: '>=12'}
 
-  /ansi-sequence-parser@1.1.0:
-    resolution: {integrity: sha512-lEm8mt52to2fT8GhciPCGeCXACSz2UwIN4X2e2LJSnZ5uAbn2/dsYdOmUXq0AtWS5cpAupysIneExOgH0Vd2TQ==}
-    dev: false
-
   /ansi-styles@3.2.1:
     resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==}
     engines: {node: '>=4'}
@@ -12095,6 +12091,47 @@ packages:
       prosemirror-view: 1.31.5
     dev: false
 
+  /prosemirror-highlight@0.3.3(prosemirror-model@1.19.2)(prosemirror-state@1.4.3)(prosemirror-view@1.31.5)(shikiji@0.9.9):
+    resolution: {integrity: sha512-tOGyPvmRKZ49ubzKmFIwiwS7CNXlU9g/D4zZLaHGzXLVNVnBrmbDOajZ4eP0lylOAWPxZN+vrFZ9DwrtyikuoA==}
+    peerDependencies:
+      '@types/hast': ^3.0.0
+      highlight.js: ^11.9.0
+      lowlight: ^3.1.0
+      prosemirror-model: ^1.19.3
+      prosemirror-state: ^1.4.3
+      prosemirror-transform: ^1.8.0
+      prosemirror-view: ^1.32.4
+      refractor: ^4.8.1
+      shiki: ^0.14.6
+      shikiji: ^0.8.0 || ^0.9.0
+    peerDependenciesMeta:
+      '@types/hast':
+        optional: true
+      highlight.js:
+        optional: true
+      lowlight:
+        optional: true
+      prosemirror-model:
+        optional: true
+      prosemirror-state:
+        optional: true
+      prosemirror-transform:
+        optional: true
+      prosemirror-view:
+        optional: true
+      refractor:
+        optional: true
+      shiki:
+        optional: true
+      shikiji:
+        optional: true
+    dependencies:
+      prosemirror-model: 1.19.2
+      prosemirror-state: 1.4.3
+      prosemirror-view: 1.31.5
+      shikiji: 0.9.9
+    dev: false
+
   /prosemirror-history@1.3.2:
     resolution: {integrity: sha512-/zm0XoU/N/+u7i5zepjmZAEnpvjDtzoPWW6VmKptcAnPadN/SStsBjMImdCEbb3seiNTpveziPTIrXQbHLtU1g==}
     dependencies:
@@ -12887,17 +12924,14 @@ packages:
     resolution: {integrity: sha512-e+/aueHx0YeIEut6RXC6K8gSf0PykwZiHD7q7AHtpTW8Kd8TpFUIWqTwhAnrGjOyOMyrwv+syr5WPagMpDpVYQ==}
     dev: true
 
-  /shiki-es@0.2.0:
+  /shikiji-core@0.9.9:
-    resolution: {integrity: sha512-RbRMD+IuJJseSZljDdne9ThrUYrwBwJR04FvN4VXpfsU3MNID5VJGHLAD5je/HGThCyEKNgH+nEkSFEWKD7C3Q==}
+    resolution: {integrity: sha512-qu5Qq7Co6JIMY312J9Ek6WYjXieeyJT/fIqmkcjF4MdnMNlUnhSqPo8/42g5UdPgdyTCwijS7Nhg8DfLSLodkg==}
     dev: false
 
-  /shiki@0.14.3:
+  /shikiji@0.9.9:
-    resolution: {integrity: sha512-U3S/a+b0KS+UkTyMjoNojvTgrBHjgp7L6ovhFVZsXmBGnVdQ4K4U9oK0z63w538S91ATngv1vXigHCSWOwnr+g==}
+    resolution: {integrity: sha512-/S3unr/0mZTstNOuAmNDEufeimtqeQb8lXvPMLsYfDvqyfmG6334bO2xmDzD0kfxH2y8gnFgSWAJpdEzksmYXg==}
     dependencies:
-      ansi-sequence-parser: 1.1.0
+      shikiji-core: 0.9.9
-      jsonc-parser: 3.2.0
-      vscode-oniguruma: 1.7.0
-      vscode-textmate: 8.0.0
     dev: false
 
   /side-channel@1.0.4:
@@ -14708,14 +14742,6 @@ packages:
     dependencies:
       vscode-languageserver-protocol: 3.16.0
 
-  /vscode-oniguruma@1.7.0:
-    resolution: {integrity: sha512-L9WMGRfrjOhgHSdOYgCt/yRMsXzLDJSL7BPrOZt73gU0iWO4mpqzqQzOz5srxqTvMBaR0XZTSrVWo4j55Rc6cA==}
-    dev: false
-
-  /vscode-textmate@8.0.0:
-    resolution: {integrity: sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==}
-    dev: false
-
   /vscode-uri@3.0.7:
     resolution: {integrity: sha512-eOpPHogvorZRobNqJGhapa0JdwaxpjVvyBp0QIUMRMSf8ZAlqOdEquKuRmw9Qwu0qXtJIWqFtMkmvJjUZmMjVA==}
 

scripts/prepare.ts

@@ -5,12 +5,6 @@ import { colorsMap } from './generate-themes'
 
 const dereference = process.platform === 'win32' ? true : undefined
 
-await fs.copy('node_modules/shiki-es/dist/assets', 'public/shiki/', {
-  dereference,
-  filter: src => src === 'node_modules/shiki/' || src.includes('languages') || src.includes('dist'),
-})
-await fs.copy('node_modules/theme-vitesse/themes', 'public/shiki/themes', { dereference })
-await fs.copy('node_modules/theme-vitesse/themes', 'node_modules/shiki/themes', { overwrite: true, dereference })
 await fs.copy(`node_modules/${iconifyEmojiPackage}/icons`, `public/emojis/${emojiPrefix}`, { overwrite: true, dereference })
 
 await fs.writeJSON('constants/themes.json', colorsMap, { spaces: 2, EOL: '\n' })

service-worker/sw.ts

@@ -39,9 +39,7 @@ if (import.meta.env.PROD) {
     /^\/oauth\//,
     /^\/signin\//,
     /^\/web-share-target\//,
-    // exclude shiki: has its own cache
+    // exclude emoji: has its own cache
-    /^\/shiki\//,
-    // exclude shiki: has its own cache
     /^\/emojis\//,
     // exclude sw: if the user navigates to it, fallback to index.html
     /^\/sw.js$/,
@@ -65,19 +63,6 @@ if (import.meta.env.PROD) {
       ],
     }),
   )
-  // include shiki cache
-  registerRoute(
-    ({ sameOrigin, url }) =>
-      sameOrigin && url.pathname.startsWith('/shiki/'),
-    new StaleWhileRevalidate({
-      cacheName: 'elk-shiki',
-      plugins: [
-        new CacheableResponsePlugin({ statuses: [200] }),
-        // 365 days max
-        new ExpirationPlugin({ purgeOnQuotaError: true, maxAgeSeconds: 60 * 60 * 24 * 365 }),
-      ],
-    }),
-  )
   // include emoji icons
   registerRoute(
     ({ sameOrigin, request, url }) =>

tests/nuxt/content-rich.test.ts

@@ -279,14 +279,6 @@ vi.mock('vue-router', async () => {
   }
 })
 
-vi.mock('shiki-es', async (importOriginal) => {
-  const mod = await importOriginal()
-  return {
-    ...(mod as any),
-    setCDN() {},
-  }
-})
-
 mockComponent('ContentMentionGroup', {
   setup(props, { slots }) {
     return () => h('mention-group', null, { default: () => slots?.default?.() })