From 6944a74653a62936b307f66c26b0c2b188ad4050 Mon Sep 17 00:00:00 2001
From: Anthony Fu <anthonyfu117@hotmail.com>
Date: Sat, 7 Jan 2023 10:31:48 +0100
Subject: [PATCH] refactor: html parsing

---
 components/content/ContentRich.setup.ts       |  20 +-
 components/status/StatusBody.vue              |  28 ++-
 composables/content-parse.ts                  | 191 ++++++++++--------
 composables/content-render.ts                 |   9 +-
 composables/emojis.ts                         |  13 ++
 config/emojis.ts                              |   1 +
 tests/__snapshots__/content-rich.test.ts.snap |  12 --
 tests/html-parse.test.ts                      |   2 +-
 8 files changed, 152 insertions(+), 124 deletions(-)

diff --git a/components/content/ContentRich.setup.ts b/components/content/ContentRich.setup.ts
index 8a8c2589..7447bb0a 100644
--- a/components/content/ContentRich.setup.ts
+++ b/components/content/ContentRich.setup.ts
@@ -4,27 +4,23 @@ defineOptions({
   name: 'ContentRich',
 })
 
-const { content, emojis, markdown = true } = defineProps<{
+const {
+  content,
+  emojis,
+  markdown = true,
+} = defineProps<{
   content: string
-  markdown?: boolean
   emojis?: Emoji[]
+  markdown?: boolean
 }>()
 
-const useEmojis = computed(() => {
-  const result: Emoji[] = []
-  if (emojis)
-    result.push(...emojis)
-
-  result.push(...currentCustomEmojis.value.emojis)
-
-  return emojisArrayToObject(result)
-})
+const emojisObject = useEmojisFallback(() => emojis)
 
 export default () => h(
   'span',
   { class: 'content-rich', dir: 'auto' },
   contentToVNode(content, {
-    emojis: useEmojis.value,
+    emojis: emojisObject.value,
     markdown,
   }),
 )
diff --git a/components/status/StatusBody.vue b/components/status/StatusBody.vue
index 75139659..446dc75d 100644
--- a/components/status/StatusBody.vue
+++ b/components/status/StatusBody.vue
@@ -1,23 +1,37 @@
 <script setup lang="ts">
 import type { Status, StatusEdit } from 'masto'
 
-const { status, withAction = true } = defineProps<{
+const {
+  status,
+  withAction = true,
+} = defineProps<{
   status: Status | StatusEdit
   withAction?: boolean
 }>()
 
 const { translation } = useTranslation(status)
+
+const emojisObject = useEmojisFallback(() => status.emojis)
+const vnode = $computed(() => {
+  if (!status.content)
+    return null
+  const vnode = contentToVNode(status.content, {
+    emojis: emojisObject.value,
+    markdown: true,
+  })
+  return vnode
+})
 </script>
 
 <template>
   <div class="status-body" whitespace-pre-wrap break-words :class="{ 'with-action': withAction }">
-    <ContentRich
+    <span
       v-if="status.content"
-      class="line-compact"
-      :content="status.content"
-      :emojis="status.emojis"
-      :lang="'language' in status && status.language"
-    />
+      class="content-rich line-compact" dir="auto"
+      :lang="('language' in status && status.language) || undefined"
+    >
+      <component :is="vnode" />
+    </span>
     <div v-else />
     <template v-if="translation.visible">
       <div my2 h-px border="b base" bg-base />
diff --git a/composables/content-parse.ts b/composables/content-parse.ts
index 9cc4bf15..47f82925 100644
--- a/composables/content-parse.ts
+++ b/composables/content-parse.ts
@@ -5,6 +5,34 @@ import { ELEMENT_NODE, TEXT_NODE, h, parse, render } from 'ultrahtml'
 import { findAndReplaceEmojisInText } from '@iconify/utils'
 import { emojiRegEx, getEmojiAttributes } from '../config/emojis'
 
+export interface ContentParseOptions {
+  emojis?: Record<string, Emoji>
+  markdown?: boolean
+  replaceUnicodeEmoji?: boolean
+  astTransforms?: Transform[]
+}
+
+const sanitizerBasicClasses = filterClasses(/^(h-\S*|p-\S*|u-\S*|dt-\S*|e-\S*|mention|hashtag|ellipsis|invisible)$/u)
+const sanitizer = sanitize({
+  // Allow basic elements as seen in https://github.com/mastodon/mastodon/blob/17f79082b098e05b68d6f0d38fabb3ac121879a9/lib/sanitize_ext/sanitize_config.rb
+  br: {},
+  p: {},
+  a: {
+    href: filterHref(),
+    class: sanitizerBasicClasses,
+    rel: set('nofollow noopener noreferrer'),
+    target: set('_blank'),
+  },
+  span: {
+    class: sanitizerBasicClasses,
+  },
+  // Allow elements potentially created for Markdown code blocks above
+  pre: {},
+  code: {
+    class: filterClasses(/^language-\w+$/),
+  },
+})
+
 const decoder = process.client ? document.createElement('textarea') : null
 export function decodeHtml(text: string) {
   if (!decoder)
@@ -18,11 +46,19 @@ export function decodeHtml(text: string) {
  * Parse raw HTML form Mastodon server to AST,
  * with interop of custom emojis and inline Markdown syntax
  */
-export function parseMastodonHTML(html: string, customEmojis: Record<string, Emoji> = {}, markdown = true, forTiptap = false) {
+export function parseMastodonHTML(
+  html: string,
+  options: ContentParseOptions = {},
+) {
+  const {
+    markdown = true,
+    replaceUnicodeEmoji = true,
+  } = options
+
   if (markdown) {
     // Handle code blocks
     html = html
-      .replace(/>(```|~~~)(\w*)([\s\S]+?)\1/g, (_1, _2, lang, raw) => {
+      .replace(/>(```|~~~)(\w*)([\s\S]+?)\1/g, (_1, _2, lang: string, raw: string) => {
         const code = htmlToText(raw)
         const classes = lang ? ` class="language-${lang}"` : ''
         return `><pre><code${classes}>${code}</code></pre>`
@@ -30,39 +66,31 @@ export function parseMastodonHTML(html: string, customEmojis: Record<string, Emo
   }
 
   // Always sanitize the raw HTML data *after* it has been modified
-  const basicClasses = filterClasses(/^(h-\S*|p-\S*|u-\S*|dt-\S*|e-\S*|mention|hashtag|ellipsis|invisible)$/u)
-  return transformSync(parse(html), [
-    sanitize({
-      // Allow basic elements as seen in https://github.com/mastodon/mastodon/blob/17f79082b098e05b68d6f0d38fabb3ac121879a9/lib/sanitize_ext/sanitize_config.rb
-      br: {},
-      p: {},
-      a: {
-        href: filterHref(),
-        class: basicClasses,
-        rel: set('nofollow noopener noreferrer'),
-        target: set('_blank'),
-      },
-      span: {
-        class: basicClasses,
-      },
-      // Allow elements potentially created for Markdown code blocks above
-      pre: {},
-      code: {
-        class: filterClasses(/^language-\w+$/),
-      },
-    }),
-    // Unicode emojis to images, but only if not converting HTML for Tiptap
-    !forTiptap ? replaceUnicodeEmoji() : noopTransform(),
-    markdown ? formatMarkdown() : noopTransform(),
-    replaceCustomEmoji(customEmojis),
-  ])
+  const transforms: Transform[] = [
+    sanitizer,
+    ...options.astTransforms || [],
+  ]
+
+  if (replaceUnicodeEmoji)
+    transforms.push(transformUnicodeEmoji)
+
+  if (markdown)
+    transforms.push(transformMarkdown)
+
+  transforms.push(replaceCustomEmoji(options.emojis || {}))
+
+  return transformSync(parse(html), transforms)
 }
 
 /**
  * Converts raw HTML form Mastodon server to HTML for Tiptap editor
  */
 export function convertMastodonHTML(html: string, customEmojis: Record<string, Emoji> = {}) {
-  const tree = parseMastodonHTML(html, customEmojis, true, true)
+  const tree = parseMastodonHTML(html, {
+    emojis: customEmojis,
+    markdown: true,
+    replaceUnicodeEmoji: false,
+  })
   return render(tree)
 }
 
@@ -162,11 +190,6 @@ function transformSync(doc: Node, transforms: Transform[]) {
   return doc
 }
 
-// A transformation that does nothing. Useful for conditional transform chains.
-function noopTransform(): Transform {
-  return node => node
-}
-
 // A tree transform for sanitizing elements & their attributes.
 type AttrSanitizers = Record<string, (value: string | undefined) => string | undefined>
 function sanitize(allowedElements: Record<string, AttrSanitizers>): Transform {
@@ -241,27 +264,25 @@ function filterHref() {
   }
 }
 
-function replaceUnicodeEmoji(): Transform {
-  return (node) => {
-    if (node.type !== TEXT_NODE)
-      return node
+function transformUnicodeEmoji(node: Node) {
+  if (node.type !== TEXT_NODE)
+    return node
 
-    let start = 0
+  let start = 0
 
-    const matches = [] as (string | Node)[]
-    findAndReplaceEmojisInText(emojiRegEx, node.value, (match, result) => {
-      const attrs = getEmojiAttributes(match)
-      matches.push(result.slice(start))
-      matches.push(h('img', { src: attrs.src, alt: attrs.alt, class: attrs.class }))
-      start = result.length + match.match.length
-      return undefined
-    })
-    if (matches.length === 0)
-      return node
+  const matches = [] as (string | Node)[]
+  findAndReplaceEmojisInText(emojiRegEx, node.value, (match, result) => {
+    const attrs = getEmojiAttributes(match)
+    matches.push(result.slice(start))
+    matches.push(h('img', { src: attrs.src, alt: attrs.alt, class: attrs.class }))
+    start = result.length + match.match.length
+    return undefined
+  })
+  if (matches.length === 0)
+    return node
 
-    matches.push(node.value.slice(start))
-    return matches.filter(Boolean)
-  }
+  matches.push(node.value.slice(start))
+  return matches.filter(Boolean)
 }
 
 function replaceCustomEmoji(customEmojis: Record<string, Emoji>): Transform {
@@ -286,47 +307,45 @@ function replaceCustomEmoji(customEmojis: Record<string, Emoji>): Transform {
   }
 }
 
-function formatMarkdown(): Transform {
-  const replacements: [RegExp, (c: (string | Node)[]) => Node][] = [
-    [/\*\*\*(.*?)\*\*\*/g, c => h('b', null, [h('em', null, c)])],
-    [/\*\*(.*?)\*\*/g, c => h('b', null, c)],
-    [/\*(.*?)\*/g, c => h('em', null, c)],
-    [/~~(.*?)~~/g, c => h('del', null, c)],
-    [/`([^`]+?)`/g, c => h('code', null, c)],
-  ]
+const _markdownReplacements: [RegExp, (c: (string | Node)[]) => Node][] = [
+  [/\*\*\*(.*?)\*\*\*/g, c => h('b', null, [h('em', null, c)])],
+  [/\*\*(.*?)\*\*/g, c => h('b', null, c)],
+  [/\*(.*?)\*/g, c => h('em', null, c)],
+  [/~~(.*?)~~/g, c => h('del', null, c)],
+  [/`([^`]+?)`/g, c => h('code', null, c)],
+]
 
-  function process(value: string) {
-    const results = [] as (string | Node)[]
+function _markdownProcess(value: string) {
+  const results = [] as (string | Node)[]
 
-    let start = 0
-    while (true) {
-      let found: { match: RegExpMatchArray; replacer: (c: (string | Node)[]) => Node } | undefined
+  let start = 0
+  while (true) {
+    let found: { match: RegExpMatchArray; replacer: (c: (string | Node)[]) => Node } | undefined
 
-      for (const [re, replacer] of replacements) {
-        re.lastIndex = start
+    for (const [re, replacer] of _markdownReplacements) {
+      re.lastIndex = start
 
-        const match = re.exec(value)
-        if (match) {
-          if (!found || match.index < found.match.index!)
-            found = { match, replacer }
-        }
+      const match = re.exec(value)
+      if (match) {
+        if (!found || match.index < found.match.index!)
+          found = { match, replacer }
       }
-
-      if (!found)
-        break
-
-      results.push(value.slice(start, found.match.index))
-      results.push(found.replacer(process(found.match[1])))
-      start = found.match.index! + found.match[0].length
     }
 
-    results.push(value.slice(start))
-    return results.filter(Boolean)
+    if (!found)
+      break
+
+    results.push(value.slice(start, found.match.index))
+    results.push(found.replacer(_markdownProcess(found.match[1])))
+    start = found.match.index! + found.match[0].length
   }
 
-  return (node) => {
-    if (node.type !== TEXT_NODE)
-      return node
-    return process(node.value)
-  }
+  results.push(value.slice(start))
+  return results.filter(Boolean)
+}
+
+function transformMarkdown(node: Node) {
+  if (node.type !== TEXT_NODE)
+    return node
+  return _markdownProcess(node.value)
 }
diff --git a/composables/content-render.ts b/composables/content-render.ts
index c4938a87..31cd980a 100644
--- a/composables/content-render.ts
+++ b/composables/content-render.ts
@@ -1,9 +1,9 @@
-import type { Emoji } from 'masto'
 import { TEXT_NODE } from 'ultrahtml'
 import type { Node } from 'ultrahtml'
 import { Fragment, h, isVNode } from 'vue'
 import type { VNode } from 'vue'
 import { RouterLink } from 'vue-router'
+import type { ContentParseOptions } from './content-parse'
 import { decodeHtml, parseMastodonHTML } from './content-parse'
 import ContentCode from '~/components/content/ContentCode.vue'
 import AccountHoverWrapper from '~/components/account/AccountHoverWrapper.vue'
@@ -13,12 +13,9 @@ import AccountHoverWrapper from '~/components/account/AccountHoverWrapper.vue'
 */
 export function contentToVNode(
   content: string,
-  { emojis = {}, markdown = true }: {
-    emojis?: Record<string, Emoji>
-    markdown?: boolean
-  } = {},
+  options?: ContentParseOptions,
 ): VNode {
-  const tree = parseMastodonHTML(content, emojis, markdown)
+  const tree = parseMastodonHTML(content, options)
   return h(Fragment, (tree.children as Node[]).map(n => treeToVNode(n)))
 }
 
diff --git a/composables/emojis.ts b/composables/emojis.ts
index 5e466c16..243c7bd3 100644
--- a/composables/emojis.ts
+++ b/composables/emojis.ts
@@ -51,3 +51,16 @@ export const customEmojisData = computed(() => currentCustomEmojis.value.emojis.
       emojis: transformEmojiData(currentCustomEmojis.value.emojis),
     }]
   : undefined)
+
+export function useEmojisFallback(emojisGetter: () => Emoji[] | undefined) {
+  return computed(() => {
+    const result: Emoji[] = []
+    const emojis = emojisGetter()
+    if (emojis)
+      result.push(...emojis)
+
+    result.push(...currentCustomEmojis.value.emojis)
+
+    return emojisArrayToObject(result)
+  })
+}
diff --git a/config/emojis.ts b/config/emojis.ts
index 6bc6dc82..a1fa98dc 100644
--- a/config/emojis.ts
+++ b/config/emojis.ts
@@ -1,3 +1,4 @@
+// @unimport-disabled
 import { emojiFilename, emojiPrefix, emojiRegEx } from '@iconify-emoji/twemoji'
 import type { EmojiRegexMatch } from '@iconify/utils/lib/emoji/replace/find'
 import { getEmojiMatchesInText } from '@iconify/utils/lib/emoji/replace/find'
diff --git a/tests/__snapshots__/content-rich.test.ts.snap b/tests/__snapshots__/content-rich.test.ts.snap
index dcc39728..7c0bbf42 100644
--- a/tests/__snapshots__/content-rich.test.ts.snap
+++ b/tests/__snapshots__/content-rich.test.ts.snap
@@ -1,12 +1,5 @@
 // Vitest Snapshot v1
 
-exports[`content-rich > JavaScript hrefs get removed 1`] = `
-"<p>
-  <a href=\\"#\\" rel=\\"nofollow noopener noreferrer\\" target=\\"_blank\\">click me</a>
-</p>
-"
-`;
-
 exports[`content-rich > code frame 1`] = `
 "<p>Testing code block</p><p><pre lang=\\"ts\\">import { useMouse, usePreferredDark } from &#39;@vueuse/core&#39;
 // tracks mouse position
@@ -75,8 +68,3 @@ exports[`content-rich > link + mention 1`] = `
 </p>
 "
 `;
-
-exports[`content-rich > script tags get removed 1`] = `
-"<p></p>
-"
-`;
diff --git a/tests/html-parse.test.ts b/tests/html-parse.test.ts
index 04c0a684..5c61fc6c 100644
--- a/tests/html-parse.test.ts
+++ b/tests/html-parse.test.ts
@@ -67,7 +67,7 @@ describe('html-parse', () => {
 })
 
 async function render(input: string, emojis?: Record<string, Emoji>) {
-  const tree = parseMastodonHTML(input, emojis)
+  const tree = parseMastodonHTML(input, { emojis })
   const html = await renderTree(tree)
   let formatted = ''
   const serializedText = treeToText(tree).trim()