Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 20e6c8b

Browse filesBrowse files
antfuantfu
authored
feat(shortcuts): guess embedded language and auto load in shortcuts (#932)
1 parent db6910f commit 20e6c8b
Copy full SHA for 20e6c8b

File tree

Expand file treeCollapse file tree

13 files changed

+198
-39
lines changed
Filter options
Expand file treeCollapse file tree

13 files changed

+198
-39
lines changed

‎packages/core/src/constructors/bundle-factory.ts

Copy file name to clipboardExpand all lines: packages/core/src/constructors/bundle-factory.ts
+31-20Lines changed: 31 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,12 @@ export function createdBundledHighlighter<BundledLangs extends string, BundledTh
134134
loadTheme(...themes) {
135135
return core.loadTheme(...themes.map(resolveTheme))
136136
},
137+
getBundledLanguages() {
138+
return bundledLanguages
139+
},
140+
getBundledThemes() {
141+
return bundledThemes
142+
},
137143
}
138144
}
139145

@@ -224,53 +230,58 @@ export function makeSingletonHighlighter<L extends string, T extends string>(
224230
return getSingletonHighlighter
225231
}
226232

233+
export interface CreateSingletonShorthandsOptions<L extends string, T extends string> {
234+
/**
235+
* A custom function to guess embedded languages to be loaded.
236+
*/
237+
guessEmbeddedLanguages?: (code: string, lang: string | undefined, highlighter: HighlighterGeneric<L, T>) => Awaitable<string[] | undefined>
238+
}
239+
227240
export function createSingletonShorthands<L extends string, T extends string>(
228241
createHighlighter: CreateHighlighterFactory<L, T>,
242+
config?: CreateSingletonShorthandsOptions<L, T>,
229243
): ShorthandsBundle<L, T> {
230244
const getSingletonHighlighter = makeSingletonHighlighter(createHighlighter)
231245

246+
async function get(code: string, options: CodeToTokensOptions<L, T> | CodeToHastOptions<L, T>): Promise<HighlighterGeneric<L, T>> {
247+
const shiki = await getSingletonHighlighter({
248+
langs: [options.lang as L],
249+
themes: ('theme' in options ? [options.theme] : Object.values(options.themes)) as T[],
250+
})
251+
const langs = await config?.guessEmbeddedLanguages?.(code, options.lang, shiki) as L[]
252+
if (langs) {
253+
await shiki.loadLanguage(...langs)
254+
}
255+
return shiki
256+
}
257+
232258
return {
233259
getSingletonHighlighter(options) {
234260
return getSingletonHighlighter(options)
235261
},
236262

237263
async codeToHtml(code, options) {
238-
const shiki = await getSingletonHighlighter({
239-
langs: [options.lang as L],
240-
themes: ('theme' in options ? [options.theme] : Object.values(options.themes)) as T[],
241-
})
264+
const shiki = await get(code, options)
242265
return shiki.codeToHtml(code, options)
243266
},
244267

245268
async codeToHast(code, options) {
246-
const shiki = await getSingletonHighlighter({
247-
langs: [options.lang as L],
248-
themes: ('theme' in options ? [options.theme] : Object.values(options.themes)) as T[],
249-
})
269+
const shiki = await get(code, options)
250270
return shiki.codeToHast(code, options)
251271
},
252272

253273
async codeToTokens(code, options) {
254-
const shiki = await getSingletonHighlighter({
255-
langs: [options.lang as L],
256-
themes: ('theme' in options ? [options.theme] : Object.values(options.themes)) as T[],
257-
})
274+
const shiki = await get(code, options)
258275
return shiki.codeToTokens(code, options)
259276
},
260277

261278
async codeToTokensBase(code, options) {
262-
const shiki = await getSingletonHighlighter({
263-
langs: [options.lang as L],
264-
themes: [options.theme as T],
265-
})
279+
const shiki = await get(code, options)
266280
return shiki.codeToTokensBase(code, options)
267281
},
268282

269283
async codeToTokensWithThemes(code, options) {
270-
const shiki = await getSingletonHighlighter({
271-
langs: [options.lang as L],
272-
themes: Object.values(options.themes).filter(Boolean) as T[],
273-
})
284+
const shiki = await get(code, options)
274285
return shiki.codeToTokensWithThemes(code, options)
275286
},
276287

‎packages/core/src/constructors/highlighter.ts

Copy file name to clipboardExpand all lines: packages/core/src/constructors/highlighter.ts
+4Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ export async function createHighlighterCore(options: HighlighterCoreOptions<fals
2626
codeToTokens: (code, options) => codeToTokens(internal, code, options),
2727
codeToHast: (code, options) => codeToHast(internal, code, options),
2828
codeToHtml: (code, options) => codeToHtml(internal, code, options),
29+
getBundledLanguages: () => ({}),
30+
getBundledThemes: () => ({}),
2931
...internal,
3032
getInternalContext: () => internal,
3133
}
@@ -49,6 +51,8 @@ export function createHighlighterCoreSync(options: HighlighterCoreOptions<true>)
4951
codeToTokens: (code, options) => codeToTokens(internal, code, options),
5052
codeToHast: (code, options) => codeToHast(internal, code, options),
5153
codeToHtml: (code, options) => codeToHtml(internal, code, options),
54+
getBundledLanguages: () => ({}),
55+
getBundledThemes: () => ({}),
5256
...internal,
5357
getInternalContext: () => internal,
5458
}

‎packages/core/src/highlight/code-to-tokens-base.ts

Copy file name to clipboardExpand all lines: packages/core/src/highlight/code-to-tokens-base.ts
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ function _tokenizeWithTheme(
185185
let tokensWithScopesIndex
186186

187187
if (options.includeExplanation) {
188-
resultWithScopes = grammar.tokenizeLine(line, stateStack)
188+
resultWithScopes = grammar.tokenizeLine(line, stateStack, tokenizeTimeLimit)
189189
tokensWithScopes = resultWithScopes.tokens
190190
tokensWithScopesIndex = 0
191191
}

‎packages/core/src/textmate/registry.ts

Copy file name to clipboardExpand all lines: packages/core/src/textmate/registry.ts
-4Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,6 @@ export class Registry extends TextMateRegistry {
6767
this._textmateThemeCache.set(theme, textmateTheme)
6868
}
6969

70-
// @ts-expect-error Access private `_syncRegistry`, but should work in runtime
7170
this._syncRegistry.setTheme(textmateTheme)
7271
}
7372

@@ -100,7 +99,6 @@ export class Registry extends TextMateRegistry {
10099
unbalancedBracketSelectors: lang.unbalancedBracketSelectors || [],
101100
}
102101

103-
// @ts-expect-error Private members, set this to override the previous grammar (that can be a stub)
104102
this._syncRegistry._rawGrammars.set(lang.scopeName, lang)
105103
const g = this.loadGrammarWithConfiguration(lang.scopeName, 1, grammarConfig) as Grammar
106104
g.name = lang.name
@@ -119,9 +117,7 @@ export class Registry extends TextMateRegistry {
119117
this._resolvedGrammars.delete(e.name)
120118
// Reset cache
121119
this._loadedLanguagesCache = null
122-
// @ts-expect-error clear cache
123120
this._syncRegistry?._injectionGrammars?.delete(e.scopeName)
124-
// @ts-expect-error clear cache
125121
this._syncRegistry?._grammars?.delete(e.scopeName)
126122
this.loadLanguage(this._langMap.get(e.name)!)
127123
}

‎packages/shiki/src/bundle-full.ts

Copy file name to clipboardExpand all lines: packages/shiki/src/bundle-full.ts
+2Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import type { BundledLanguage } from './langs-bundle-full'
44
import type { BundledTheme } from './themes'
55
import { createdBundledHighlighter, createSingletonShorthands, warnDeprecated } from './core'
66
import { createOnigurumaEngine } from './engine-oniguruma'
7+
import { guessEmbeddedLanguages } from './guess'
78
import { bundledLanguages } from './langs-bundle-full'
89
import { bundledThemes } from './themes'
910

@@ -46,6 +47,7 @@ export const {
4647
BundledTheme
4748
>(
4849
createHighlighter,
50+
{ guessEmbeddedLanguages },
4951
)
5052

5153
/**

‎packages/shiki/src/bundle-web.ts

Copy file name to clipboardExpand all lines: packages/shiki/src/bundle-web.ts
+2Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import type { BundledLanguage } from './langs-bundle-web'
44
import type { BundledTheme } from './themes'
55
import { createdBundledHighlighter, createSingletonShorthands, warnDeprecated } from './core'
66
import { createOnigurumaEngine } from './engine-oniguruma'
7+
import { guessEmbeddedLanguages } from './guess'
78
import { bundledLanguages } from './langs-bundle-web'
89
import { bundledThemes } from './themes'
910

@@ -46,6 +47,7 @@ export const {
4647
BundledTheme
4748
>(
4849
createHighlighter,
50+
{ guessEmbeddedLanguages },
4951
)
5052

5153
/**

‎packages/shiki/src/guess.ts

Copy file name to clipboard
+26Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
import type { HighlighterGeneric } from '@shikijs/types'
2+
3+
export function guessEmbeddedLanguages(
4+
code: string,
5+
_lang: string | undefined,
6+
shiki: HighlighterGeneric<any, any>,
7+
): string[] {
8+
const langs = new Set<string>()
9+
// For HTML code blocks like Vue SFC
10+
for (const match of code.matchAll(/lang=["']([\w-]+)["']/g)) {
11+
langs.add(match[1])
12+
}
13+
// For markdown code blocks
14+
for (const match of code.matchAll(/(?:```|~~~)([\w-]+)/g)) {
15+
langs.add(match[1])
16+
}
17+
// For latex
18+
for (const match of code.matchAll(/\\begin\{([\w-]+)\}/g)) {
19+
langs.add(match[1])
20+
}
21+
22+
// Only include known languages
23+
const bundle = shiki.getBundledLanguages()
24+
return Array.from(langs)
25+
.filter(l => l && bundle[l])
26+
}

‎packages/shiki/test/out/shorthand-markdown1.html

Copy file name to clipboardExpand all lines: packages/shiki/test/out/shorthand-markdown1.html
+14Lines changed: 14 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/shiki/test/out/shorthand-markdown2.html

Copy file name to clipboardExpand all lines: packages/shiki/test/out/shorthand-markdown2.html
+20Lines changed: 20 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/shiki/test/shorthands-markdown.test.ts

Copy file name to clipboard
+76Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
import { expect, it } from 'vitest'
2+
import { codeToHtml, getSingletonHighlighter } from '../src'
3+
4+
const inputMarkdown1 = `
5+
This is a markdown file
6+
7+
\`\`\`js
8+
console.log("hello")
9+
\`\`\`
10+
11+
~~~pug
12+
div
13+
p hello
14+
~~~
15+
16+
Even those grammars in markdown are lazy loaded, \`codeToHtml\` shorthand should capture them and load automatically.
17+
`
18+
19+
const inputMarkdown2 = `
20+
Some other languages
21+
22+
\`\`\`js
23+
console.log("hello")
24+
\`\`\`
25+
26+
~~~python
27+
print("hello")
28+
~~~
29+
30+
\`\`\`html
31+
<div class="foo">bar</div>
32+
<style>
33+
.foo {
34+
color: red;
35+
}
36+
</style>
37+
\`\`\`
38+
`
39+
40+
it('codeToHtml', async () => {
41+
const highlighter = await getSingletonHighlighter()
42+
expect(highlighter.getLoadedLanguages())
43+
.toEqual([])
44+
45+
await expect(await codeToHtml(inputMarkdown1, { lang: 'markdown', theme: 'vitesse-light' }))
46+
.toMatchFileSnapshot(`out/shorthand-markdown1.html`)
47+
48+
expect.soft(highlighter.getLoadedLanguages())
49+
.toContain('javascript')
50+
expect.soft(highlighter.getLoadedLanguages())
51+
.toContain('pug')
52+
53+
await expect(await codeToHtml(inputMarkdown2, { lang: 'markdown', theme: 'vitesse-light' }))
54+
.toMatchFileSnapshot(`out/shorthand-markdown2.html`)
55+
56+
expect.soft(highlighter.getLoadedLanguages())
57+
.toContain('python')
58+
expect.soft(highlighter.getLoadedLanguages())
59+
.toContain('html')
60+
61+
expect.soft(highlighter.getLoadedLanguages())
62+
.toMatchInlineSnapshot(`
63+
[
64+
"javascript",
65+
"css",
66+
"html",
67+
"pug",
68+
"python",
69+
"markdown",
70+
"md",
71+
"js",
72+
"jade",
73+
"py",
74+
]
75+
`)
76+
})

‎packages/types/src/highlighter.ts

Copy file name to clipboardExpand all lines: packages/types/src/highlighter.ts
+8Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,14 @@ export interface HighlighterGeneric<BundledLangKeys extends string, BundledTheme
143143
* @deprecated
144144
*/
145145
getInternalContext: () => ShikiInternal
146+
/**
147+
* Get bundled languages object
148+
*/
149+
getBundledLanguages: () => Record<BundledLangKeys, LanguageInput>
150+
/**
151+
* Get bundled themes object
152+
*/
153+
getBundledThemes: () => Record<BundledThemeKeys, ThemeInput>
146154
}
147155

148156
/**

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.