OurBigBook Docs
marked/test/unit/Lexer.test.js
import { Lexer } from '../../lib/marked.esm.js';
import { describe, it } from 'node:test';
import assert from 'node:assert';
function expectTokens({ md, options, tokens = [], links = {}, log = false }) {
const lexer = new Lexer(options);
const actual = lexer.lex(md);
const expected = tokens;
expected.links = links;
if (log) {
console.log(JSON.stringify(
actual,
(k, v) => v === undefined ? null : v,
2,
));
}
assert.deepEqual(actual, expected);
}
function expectInlineTokens({ md, options, tokens, links = {} }) {
const lexer = new Lexer(options);
lexer.tokens.links = links;
const outTokens = [];
lexer.inlineTokens(md, outTokens);
assert.deepEqual(outTokens, tokens);
}
describe('Lexer', () => {
describe('rules', () => {
it('should create indentation regexes for zero indent', () => {
const {
nextBulletRegex,
hrRegex,
fencesBeginRegex,
headingBeginRegex,
htmlBeginRegex,
blockquoteBeginRegex,
} = new Lexer().tokenizer.rules.other;
assert.ok(nextBulletRegex(0).test('- item\n'));
assert.ok(hrRegex(0).test('---\n'));
assert.ok(fencesBeginRegex(0).test('```'));
assert.ok(headingBeginRegex(0).test('# heading'));
assert.ok(htmlBeginRegex(0).test('<div>'));
assert.ok(blockquoteBeginRegex(0).test('> quote'));
});
});
describe('paragraph', () => {
it('space between paragraphs', () => {
expectTokens({
md: 'paragraph 1\n\nparagraph 2',
tokens: [
{
type: 'paragraph',
raw: 'paragraph 1',
text: 'paragraph 1',
tokens: [{ type: 'text', raw: 'paragraph 1', text: 'paragraph 1', escaped: false }],
},
{ type: 'space', raw: '\n\n' },
{
type: 'paragraph',
raw: 'paragraph 2',
text: 'paragraph 2',
tokens: [{ type: 'text', raw: 'paragraph 2', text: 'paragraph 2', escaped: false }],
},
],
});
});
});
describe('code', () => {
it('indented code', () => {
expectTokens({
md: ' code',
tokens: [
{ type: 'code', raw: ' code', text: 'code', codeBlockStyle: 'indented' },
],
});
});
it('indented code with trailing blank lines', () => {
expectTokens({
md: ' code\n \n\t\t\n \t\n\nnext',
tokens: [
{ type: 'code', raw: ' code', text: 'code', codeBlockStyle: 'indented' },
{ type: 'space', raw: '\n \n\t\t\n \t\n\n' },
{
type: 'paragraph',
raw: 'next',
text: 'next',
tokens: [{ type: 'text', raw: 'next', text: 'next', escaped: false }],
},
],
});
});
it('fenced code', () => {
expectTokens({
md: '```\ncode\n```',
tokens: [
{ type: 'code', raw: '```\ncode\n```', text: 'code', lang: '' },
],
});
});
it('fenced code lang', () => {
expectTokens({
md: '```text\ncode\n```',
tokens: [
{ type: 'code', raw: '```text\ncode\n```', text: 'code', lang: 'text' },
],
});
});
});
describe('headings', () => {
it('depth', () => {
expectTokens({
md: `
# heading 1
## heading 2
### heading 3
#### heading 4
##### heading 5
###### heading 6
lheading 1
==========
lheading 2
----------
`,
tokens: [
{
type: 'space',
raw: '\n',
},
{
type: 'heading',
raw: '# heading 1',
depth: 1,
text: 'heading 1',
tokens: [{ type: 'text', raw: 'heading 1', text: 'heading 1', escaped: false }],
},
{
type: 'space',
raw: '\n\n',
},
{
type: 'heading',
raw: '## heading 2',
depth: 2,
text: 'heading 2',
tokens: [{ type: 'text', raw: 'heading 2', text: 'heading 2', escaped: false }],
},
{
type: 'space',
raw: '\n\n',
},
{
type: 'heading',
raw: '### heading 3',
depth: 3,
text: 'heading 3',
tokens: [{ type: 'text', raw: 'heading 3', text: 'heading 3', escaped: false }],
},
{
type: 'space',
raw: '\n\n',
},
{
type: 'heading',
raw: '#### heading 4',
depth: 4,
text: 'heading 4',
tokens: [{ type: 'text', raw: 'heading 4', text: 'heading 4', escaped: false }],
},
{
type: 'space',
raw: '\n\n',
},
{
type: 'heading',
raw: '##### heading 5',
depth: 5,
text: 'heading 5',
tokens: [{ type: 'text', raw: 'heading 5', text: 'heading 5', escaped: false }],
},
{
type: 'space',
raw: '\n\n',
},
{
type: 'heading',
raw: '###### heading 6',
depth: 6,
text: 'heading 6',
tokens: [{ type: 'text', raw: 'heading 6', text: 'heading 6', escaped: false }],
},
{
type: 'space',
raw: '\n\n',
},
{
type: 'heading',
raw: 'lheading 1\n==========',
depth: 1,
text: 'lheading 1',
tokens: [{ type: 'text', raw: 'lheading 1', text: 'lheading 1', escaped: false }],
},
{
type: 'space',
raw: '\n\n',
},
{
type: 'heading',
raw: 'lheading 2\n----------\n',
depth: 2,
text: 'lheading 2',
tokens: [{ type: 'text', raw: 'lheading 2', text: 'lheading 2', escaped: false }],
},
],
});
});
it('lheading should trim trailing whitespace from text', () => {
expectTokens({
md: 'heading with trailing spaces \n==============================',
tokens: [{
type: 'heading',
raw: 'heading with trailing spaces \n==============================',
depth: 1,
text: 'heading with trailing spaces',
tokens: [{ type: 'text', raw: 'heading with trailing spaces', text: 'heading with trailing spaces', escaped: false }],
}],
});
});
it('should not be heading if depth > 6', () => {
expectTokens({
md: '####### heading 7',
tokens: [{
type: 'paragraph',
raw: '####### heading 7',
text: '####### heading 7',
tokens: [{ type: 'text', raw: '####### heading 7', text: '####### heading 7', escaped: false }],
}],
});
});
it('should strip trailing hashes with preceding space', () => {
expectTokens({
md: '## heading ##',
tokens: [{
type: 'heading',
raw: '## heading ##',
depth: 2,
text: 'heading',
tokens: [{ type: 'text', raw: 'heading', text: 'heading', escaped: false }],
}],
});
});
it('should not strip trailing hashes without preceding space', () => {
expectTokens({
md: '## heading##',
tokens: [{
type: 'heading',
raw: '## heading##',
depth: 2,
text: 'heading##',
tokens: [{ type: 'text', raw: 'heading##', text: 'heading##', escaped: false }],
}],
});
});
it('should tokenize inline content inside heading', () => {
expectTokens({
md: '## **bold** and _em_',
tokens: [{
type: 'heading',
raw: '## **bold** and _em_',
depth: 2,
text: '**bold** and _em_',
tokens: [
{
type: 'strong',
raw: '**bold**',
text: 'bold',
tokens: [{ type: 'text', raw: 'bold', text: 'bold', escaped: false }],
},
{ type: 'text', raw: ' and ', text: ' and ', escaped: false },
{
type: 'em',
raw: '_em_',
text: 'em',
tokens: [{ type: 'text', raw: 'em', text: 'em', escaped: false }],
},
],
}],
});
});
it('heading immediately followed by paragraph without blank line', () => {
expectTokens({
md: '# heading\nparagraph',
tokens: [
{
type: 'heading',
raw: '# heading\n',
depth: 1,
text: 'heading',
tokens: [{ type: 'text', raw: 'heading', text: 'heading', escaped: false }],
},
{
type: 'paragraph',
raw: 'paragraph',
text: 'paragraph',
tokens: [{ type: 'text', raw: 'paragraph', text: 'paragraph', escaped: false }],
},
],
});
});
});
describe('table', () => {
it('pipe table', () => {
expectTokens({
md: `
| a | b |
|---|---|
| 1 | 2 |
`,
tokens: [{
type: 'space',
raw: '\n',
}, {
type: 'table',
align: [null, null],
raw: '| a | b |\n|---|---|\n| 1 | 2 |\n',
header: [
{
text: 'a',
tokens: [{ type: 'text', raw: 'a', text: 'a', escaped: false }],
header: true,
align: null,
},
{
text: 'b',
tokens: [{ type: 'text', raw: 'b', text: 'b', escaped: false }],
header: true,
align: null,
},
],
rows: [
[
{
text: '1',
tokens: [{ type: 'text', raw: '1', text: '1', escaped: false }],
header: false,
align: null,
},
{
text: '2',
tokens: [{ type: 'text', raw: '2', text: '2', escaped: false }],
header: false,
align: null,
},
],
],
}],
});
});
it('table after para', () => {
expectTokens({
md: `
paragraph 1
| a | b |
|---|---|
| 1 | 2 |
`,
tokens: [{
type: 'space',
raw: '\n',
}, {
type: 'paragraph',
raw: 'paragraph 1\n',
text: 'paragraph 1',
tokens: [{ type: 'text', raw: 'paragraph 1', text: 'paragraph 1', escaped: false }],
},
{
type: 'table',
align: [null, null],
raw: '| a | b |\n|---|---|\n| 1 | 2 |\n',
header: [
{
text: 'a',
tokens: [{ type: 'text', raw: 'a', text: 'a', escaped: false }],
header: true,
align: null,
},
{
text: 'b',
tokens: [{ type: 'text', raw: 'b', text: 'b', escaped: false }],
header: true,
align: null,
},
],
rows: [
[
{
text: '1',
tokens: [{ type: 'text', raw: '1', text: '1', escaped: false }],
header: false,
align: null,
},
{
text: '2',
tokens: [{ type: 'text', raw: '2', text: '2', escaped: false }],
header: false,
align: null,
},
],
],
},
],
});
});
it('align table', () => {
expectTokens({
md: `
| a | b | c |
|:--|:-:|--:|
| 1 | 2 | 3 |
`,
tokens: [{
type: 'space',
raw: '\n',
}, {
type: 'table',
align: ['left', 'center', 'right'],
raw: '| a | b | c |\n|:--|:-:|--:|\n| 1 | 2 | 3 |\n',
header: [
{
text: 'a',
tokens: [{ type: 'text', raw: 'a', text: 'a', escaped: false }],
header: true,
align: 'left',
},
{
text: 'b',
tokens: [{ type: 'text', raw: 'b', text: 'b', escaped: false }],
header: true,
align: 'center',
},
{
text: 'c',
tokens: [{ type: 'text', raw: 'c', text: 'c', escaped: false }],
header: true,
align: 'right',
},
],
rows: [
[
{
text: '1',
tokens: [{ type: 'text', raw: '1', text: '1', escaped: false }],
header: false,
align: 'left',
},
{
text: '2',
tokens: [{ type: 'text', raw: '2', text: '2', escaped: false }],
header: false,
align: 'center',
},
{
text: '3',
tokens: [{ type: 'text', raw: '3', text: '3', escaped: false }],
header: false,
align: 'right',
},
],
],
}],
});
});
it('no pipe table', () => {
expectTokens({
md: `
a | b
--|--
1 | 2
`,
tokens: [
{
type: 'space',
raw: '\n',
}, {
type: 'table',
align: [null, null],
raw: 'a | b\n--|--\n1 | 2\n',
header: [
{
text: 'a',
tokens: [{ type: 'text', raw: 'a', text: 'a', escaped: false }],
header: true,
align: null,
},
{
text: 'b',
tokens: [{ type: 'text', raw: 'b', text: 'b', escaped: false }],
header: true,
align: null,
},
],
rows: [
[
{
text: '1',
tokens: [{ type: 'text', raw: '1', text: '1', escaped: false }],
header: false,
align: null,
},
{
text: '2',
tokens: [{ type: 'text', raw: '2', text: '2', escaped: false }],
header: false,
align: null,
},
],
],
}],
});
});
});
describe('hr', () => {
it('hr', () => {
expectTokens({
md: '---',
tokens: [
{ type: 'hr', raw: '---' },
],
});
});
});
describe('blockquote', () => {
it('start, inner-tokens, end', () => {
expectTokens({
md: '> blockquote',
tokens: [
{
type: 'blockquote',
raw: '> blockquote',
text: 'blockquote',
tokens: [{
type: 'paragraph',
raw: 'blockquote',
text: 'blockquote',
tokens: [
{ type: 'text', raw: 'blockquote', text: 'blockquote', escaped: false },
],
}],
},
],
});
});
it('trim newline in text', () => {
expectTokens({
md: '> blockquote\n',
tokens: [
{
type: 'blockquote',
raw: '> blockquote\n',
text: 'blockquote',
tokens: [{
type: 'paragraph',
raw: 'blockquote',
text: 'blockquote',
tokens: [
{ type: 'text', raw: 'blockquote', text: 'blockquote', escaped: false },
],
}],
},
],
});
});
it('paragraph token in list', () => {
expectTokens({
md: '- > blockquote',
tokens: [
{
type: 'list',
raw: '- > blockquote',
ordered: false,
start: '',
loose: false,
items: [
{
type: 'list_item',
raw: '- > blockquote',
task: false,
loose: false,
text: '> blockquote',
tokens: [
{
type: 'blockquote',
raw: '> blockquote',
tokens: [
{
type: 'paragraph',
raw: 'blockquote',
text: 'blockquote',
tokens: [
{ type: 'text', raw: 'blockquote', text: 'blockquote', escaped: false },
],
},
],
text: 'blockquote',
},
],
},
],
},
],
});
});
});
describe('list', () => {
it('unordered', () => {
expectTokens({
md: `
- item 1
- item 2
`,
tokens: [
{
type: 'space',
raw: '\n',
}, {
type: 'list',
raw: '- item 1\n- item 2\n',
ordered: false,
start: '',
loose: false,
items: [
{
type: 'list_item',
raw: '- item 1\n',
task: false,
loose: false,
text: 'item 1',
tokens: [{
type: 'text',
raw: 'item 1',
text: 'item 1',
tokens: [{ type: 'text', raw: 'item 1', text: 'item 1', escaped: false }],
}],
},
{
type: 'list_item',
raw: '- item 2',
task: false,
loose: false,
text: 'item 2',
tokens: [{
type: 'text',
raw: 'item 2',
text: 'item 2',
tokens: [{ type: 'text', raw: 'item 2', text: 'item 2', escaped: false }],
}],
},
],
},
],
});
});
it('ordered', () => {
expectTokens({
md: `
1. item 1
2. item 2
`,
tokens: [
{
type: 'space',
raw: '\n',
},
{
type: 'list',
raw: '1. item 1\n2. item 2\n',
ordered: true,
start: 1,
loose: false,
items: [
{
type: 'list_item',
raw: '1. item 1\n',
task: false,
loose: false,
text: 'item 1',
tokens: [
{
type: 'text',
raw: 'item 1',
text: 'item 1',
tokens: [
{
type: 'text',
raw: 'item 1',
text: 'item 1',
escaped: false,
},
],
},
],
},
{
type: 'list_item',
raw: '2. item 2',
task: false,
loose: false,
text: 'item 2',
tokens: [
{
type: 'text',
raw: 'item 2',
text: 'item 2',
tokens: [
{
type: 'text',
raw: 'item 2',
text: 'item 2',
escaped: false,
},
],
},
],
},
],
},
],
});
});
it('ordered with parenthesis', () => {
expectTokens({
md: `
1) item 1
2) item 2
`,
tokens: [
{
type: 'space',
raw: '\n',
},
{
type: 'list',
raw: '1) item 1\n2) item 2\n',
ordered: true,
start: 1,
loose: false,
items: [
{
type: 'list_item',
raw: '1) item 1\n',
task: false,
loose: false,
text: 'item 1',
tokens: [
{
type: 'text',
raw: 'item 1',
text: 'item 1',
tokens: [
{
type: 'text',
raw: 'item 1',
text: 'item 1',
escaped: false,
},
],
},
],
},
{
type: 'list_item',
raw: '2) item 2',
task: false,
loose: false,
text: 'item 2',
tokens: [
{
type: 'text',
raw: 'item 2',
text: 'item 2',
tokens: [
{
type: 'text',
raw: 'item 2',
text: 'item 2',
escaped: false,
},
],
},
],
},
],
},
],
});
});
it('space after list', () => {
expectTokens({
md: `
- item 1
- item 2
paragraph
`,
tokens: [
{
type: 'space',
raw: '\n',
},
{
type: 'list',
raw: '- item 1\n- item 2',
ordered: false,
start: '',
loose: false,
items: [
{
type: 'list_item',
raw: '- item 1\n',
task: false,
loose: false,
text: 'item 1',
tokens: [
{
type: 'text',
raw: 'item 1',
text: 'item 1',
tokens: [
{
type: 'text',
raw: 'item 1',
text: 'item 1',
escaped: false,
},
],
},
],
},
{
type: 'list_item',
raw: '- item 2',
task: false,
loose: false,
text: 'item 2',
tokens: [
{
type: 'text',
raw: 'item 2',
text: 'item 2',
tokens: [
{
type: 'text',
raw: 'item 2',
text: 'item 2',
escaped: false,
},
],
},
],
},
],
},
{
type: 'space',
raw: '\n\n',
},
{
type: 'paragraph',
raw: 'paragraph\n',
text: 'paragraph',
tokens: [
{
type: 'text',
raw: 'paragraph',
text: 'paragraph',
escaped: false,
},
],
},
],
});
});
it('start', () => {
expectTokens({
md: `
2. item 1
3. item 2
`,
tokens: [
{
type: 'space',
raw: '\n',
},
{
type: 'list',
raw: '2. item 1\n3. item 2\n',
ordered: true,
start: 2,
loose: false,
items: [
{
type: 'list_item',
raw: '2. item 1\n',
task: false,
loose: false,
text: 'item 1',
tokens: [
{
type: 'text',
raw: 'item 1',
text: 'item 1',
tokens: [
{
type: 'text',
raw: 'item 1',
text: 'item 1',
escaped: false,
},
],
},
],
},
{
type: 'list_item',
raw: '3. item 2',
task: false,
loose: false,
text: 'item 2',
tokens: [
{
type: 'text',
raw: 'item 2',
text: 'item 2',
tokens: [
{
type: 'text',
raw: 'item 2',
text: 'item 2',
escaped: false,
},
],
},
],
},
],
},
],
});
});
it('loose', () => {
expectTokens({
md: `
- item 1
- item 2
`,
tokens: [
{
type: 'space',
raw: '\n',
},
{
type: 'list',
raw: '- item 1\n\n- item 2\n',
ordered: false,
start: '',
loose: true,
items: [
{
type: 'list_item',
raw: '- item 1\n\n',
task: false,
loose: true,
text: 'item 1\n',
tokens: [
{
type: 'paragraph',
raw: 'item 1\n',
text: 'item 1',
tokens: [
{
type: 'text',
raw: 'item 1',
text: 'item 1',
escaped: false,
},
],
},
],
},
{
type: 'list_item',
raw: '- item 2',
task: false,
loose: true,
text: 'item 2',
tokens: [
{
type: 'paragraph',
raw: 'item 2',
text: 'item 2',
tokens: [
{
type: 'text',
raw: 'item 2',
text: 'item 2',
escaped: false,
},
],
},
],
},
],
},
],
});
});
it('end loose', () => {
expectTokens({
md: `
- item 1
- item 2
item 2a
- item 3
`,
tokens: [
{
type: 'space',
raw: '\n',
},
{
type: 'list',
raw: '- item 1\n- item 2\n\n item 2a\n- item 3\n',
ordered: false,
start: '',
loose: true,
items: [
{
type: 'list_item',
raw: '- item 1\n',
task: false,
loose: true,
text: 'item 1',
tokens: [
{
type: 'paragraph',
raw: 'item 1',
text: 'item 1',
tokens: [
{
type: 'text',
raw: 'item 1',
text: 'item 1',
escaped: false,
},
],
},
],
},
{
type: 'list_item',
raw: '- item 2\n\n item 2a\n',
task: false,
loose: true,
text: 'item 2\n\nitem 2a',
tokens: [
{
type: 'paragraph',
raw: 'item 2',
text: 'item 2',
tokens: [
{
type: 'text',
raw: 'item 2',
text: 'item 2',
escaped: false,
},
],
},
{
type: 'space',
raw: '\n\n',
},
{
type: 'paragraph',
raw: 'item 2a',
text: 'item 2a',
tokens: [
{
type: 'text',
raw: 'item 2a',
text: 'item 2a',
escaped: false,
},
],
},
],
},
{
type: 'list_item',
raw: '- item 3',
task: false,
loose: true,
text: 'item 3',
tokens: [
{
type: 'paragraph',
raw: 'item 3',
text: 'item 3',
tokens: [
{
type: 'text',
raw: 'item 3',
text: 'item 3',
escaped: false,
},
],
},
],
},
],
},
],
});
});
it('not loose with spaces', () => {
expectTokens({
md: `
- item 1
- item 2
`,
tokens: [
{
type: 'space',
raw: '\n',
},
{
type: 'list',
raw: '- item 1\n - item 2\n',
ordered: false,
start: '',
loose: false,
items: [
{
type: 'list_item',
raw: '- item 1\n - item 2',
task: false,
loose: false,
text: 'item 1\n- item 2',
tokens: [
{
type: 'text',
raw: 'item 1\n',
text: 'item 1',
tokens: [
{
type: 'text',
raw: 'item 1',
text: 'item 1',
escaped: false,
},
],
},
{
type: 'list',
raw: '- item 2',
ordered: false,
start: '',
loose: false,
items: [
{
type: 'list_item',
raw: '- item 2',
task: false,
loose: false,
text: 'item 2',
tokens: [
{
type: 'text',
raw: 'item 2',
text: 'item 2',
tokens: [
{
type: 'text',
raw: 'item 2',
text: 'item 2',
escaped: false,
},
],
},
],
},
],
},
],
},
],
},
],
});
});
it('task', () => {
expectTokens({
md: `
- [ ] item 1
- [x] item 2
`,
tokens: [
{
type: 'space',
raw: '\n',
},
{
type: 'list',
raw: '- [ ] item 1\n- [x] item 2\n',
ordered: false,
start: '',
loose: false,
items: [
{
type: 'list_item',
raw: '- [ ] item 1\n',
task: true,
checked: false,
loose: false,
text: 'item 1',
tokens: [
{
type: 'checkbox',
raw: '[ ] ',
checked: false,
},
{
type: 'text',
raw: 'item 1',
text: 'item 1',
tokens: [
{
type: 'text',
raw: 'item 1',
text: 'item 1',
escaped: false,
},
],
},
],
},
{
type: 'list_item',
raw: '- [x] item 2',
task: true,
checked: true,
loose: false,
text: 'item 2',
tokens: [
{
type: 'checkbox',
raw: '[x] ',
checked: true,
},
{
type: 'text',
raw: 'item 2',
text: 'item 2',
tokens: [
{
type: 'text',
raw: 'item 2',
text: 'item 2',
escaped: false,
},
],
},
],
},
],
},
],
});
});
it('does not parse setext heading text as a task checkbox', () => {
expectTokens({
md: '- [x] title\n ---',
tokens: [
{
type: 'list',
raw: '- [x] title\n ---',
ordered: false,
start: '',
loose: false,
items: [
{
type: 'list_item',
raw: '- [x] title\n ---',
task: false,
loose: false,
text: '[x] title\n---',
tokens: [
{
type: 'heading',
raw: '[x] title\n---',
depth: 2,
text: '[x] title',
tokens: [
{
type: 'text',
raw: '[x] title',
text: '[x] title',
},
],
},
],
},
],
},
],
});
});
it('keeps setext heading task candidates loose when separated by blank lines', () => {
expectTokens({
md: '- [x] title\n ---\n\n body\n- second',
tokens: [
{
type: 'list',
raw: '- [x] title\n ---\n\n body\n- second',
ordered: false,
start: '',
loose: true,
items: [
{
type: 'list_item',
raw: '- [x] title\n ---\n\n body\n',
task: false,
loose: true,
text: '[x] title\n---\n\nbody',
tokens: [
{
type: 'heading',
raw: '[x] title\n---',
depth: 2,
text: '[x] title',
tokens: [
{
type: 'text',
raw: '[x] title',
text: '[x] title',
},
],
},
{
type: 'space',
raw: '\n\n',
},
{
type: 'paragraph',
raw: 'body',
text: 'body',
tokens: [
{
type: 'text',
raw: 'body',
text: 'body',
escaped: false,
},
],
},
],
},
{
type: 'list_item',
raw: '- second',
task: false,
loose: true,
text: 'second',
tokens: [
{
type: 'paragraph',
raw: 'second',
text: 'second',
tokens: [
{
type: 'text',
raw: 'second',
text: 'second',
escaped: false,
},
],
},
],
},
],
},
],
});
});
it('multiline', () => {
expectTokens({
md: `
- line 1
line 2
`,
tokens: [
{
type: 'space',
raw: '\n',
}, {
type: 'list',
raw: '- line 1\n line 2\n',
ordered: false,
start: '',
loose: false,
items: [
{
type: 'list_item',
raw: '- line 1\n line 2',
task: false,
loose: false,
text: 'line 1\nline 2',
tokens: [{
type: 'text',
raw: 'line 1\nline 2',
text: 'line 1\nline 2',
tokens: [{ type: 'text', raw: 'line 1\nline 2', text: 'line 1\nline 2', escaped: false }],
}],
},
],
},
],
});
});
it('indented code after paragraph', () => {
expectTokens({
md: `
- a
- b
`,
tokens: [{
type: 'space',
raw: '\n',
},
{
type: 'list',
raw: '- a\n - b\n',
ordered: false,
start: '',
loose: false,
items: [
{
type: 'list_item',
raw: '- a\n - b',
task: false,
loose: false,
text: 'a\n - b',
tokens: [
{
type: 'text',
raw: 'a\n - b',
text: 'a\n- b',
tokens: [
{ type: 'text', raw: 'a\n- b', text: 'a\n- b', escaped: false },
],
},
],
},
],
},
],
});
});
it('def after paragraph', () => {
expectTokens({
md: `
- hello
[1]: hello
`,
tokens: [
{ type: 'space', raw: '\n' },
{
type: 'list',
raw: '- hello\n[1]: hello\n',
ordered: false,
start: '',
loose: false,
items: [
{
type: 'list_item',
raw: '- hello\n[1]: hello',
task: false,
loose: false,
text: 'hello\n[1]: hello',
tokens: [
{
type: 'text',
raw: 'hello\n[1]: hello',
text: 'hello\n[1]: hello',
tokens: [
{ type: 'text', raw: 'hello\n[1]: hello', text: 'hello\n[1]: hello', escaped: false },
],
},
],
},
],
},
],
});
});
});
describe('html', () => {
it('div', () => {
expectTokens({
md: '<div>html</div>',
tokens: [
{
type: 'html',
raw: '<div>html</div>',
pre: false,
block: true,
text: '<div>html</div>',
},
],
});
});
it('pre', () => {
expectTokens({
md: '<pre>html</pre>',
tokens: [
{
type: 'html',
raw: '<pre>html</pre>',
pre: true,
block: true,
text: '<pre>html</pre>',
},
],
});
});
it('html with trailing blank lines', () => {
expectTokens({
md: '<div>\n html\n</div>\n \n\t\n\nnext',
tokens: [
{
type: 'html',
raw: '<div>\n html\n</div>',
pre: false,
block: true,
text: '<div>\n html\n</div>',
},
{
type: 'space',
raw: '\n \n\t\n\n',
},
{
type: 'paragraph',
raw: 'next',
text: 'next',
tokens: [{ type: 'text', raw: 'next', text: 'next', escaped: false }],
},
],
});
});
});
describe('def', () => {
it('link', () => {
expectTokens({
md: '[link]: https://example.com',
links: {
link: { href: 'https://example.com', title: undefined },
},
tokens: [
{
type: 'def',
raw: '[link]: https://example.com',
tag: 'link',
href: 'https://example.com',
title: undefined,
},
],
});
});
it('title', () => {
expectTokens({
md: '[link]: https://example.com "title"',
links: {
link: { href: 'https://example.com', title: 'title' },
},
tokens: [
{
type: 'def',
raw: '[link]: https://example.com "title"',
tag: 'link',
href: 'https://example.com',
title: 'title',
},
],
});
});
});
describe('inline', () => {
describe('inlineTokens', () => {
it('escape', () => {
expectInlineTokens({
md: '\\>',
tokens: [
{ type: 'escape', raw: '\\>', text: '>' },
],
});
});
it('escaped punctuation inside emphasis', () => {
expectInlineTokens({
md: '**strong text\\[**\\]',
tokens: [
{
type: 'strong',
raw: '**strong text\\[**',
text: 'strong text\\[',
tokens: [
{ type: 'text', raw: 'strong text', text: 'strong text', escaped: false },
{ type: 'escape', raw: '\\[', text: '[' },
],
},
{ type: 'escape', raw: '\\]', text: ']' },
],
});
expectInlineTokens({
md: '_em\\<pha\\>sis_',
tokens: [
{
type: 'em',
raw: '_em\\<pha\\>sis_',
text: 'em\\<pha\\>sis',
tokens: [
{ type: 'text', raw: 'em', text: 'em', escaped: false },
{ type: 'escape', raw: '\\<', text: '<' },
{ type: 'text', raw: 'pha', text: 'pha', escaped: false },
{ type: 'escape', raw: '\\>', text: '>' },
{ type: 'text', raw: 'sis', text: 'sis', escaped: false },
],
},
],
});
});
it('html', () => {
expectInlineTokens({
md: '<div>html</div>',
tokens: [
{ type: 'html', raw: '<div>', inLink: false, inRawBlock: false, block: false, text: '<div>' },
{ type: 'text', raw: 'html', text: 'html', escaped: false },
{ type: 'html', raw: '</div>', inLink: false, inRawBlock: false, block: false, text: '</div>' },
],
});
});
it('link', () => {
expectInlineTokens({
md: '[link](https://example.com)',
tokens: [
{
type: 'link',
raw: '[link](https://example.com)',
href: 'https://example.com',
title: null,
text: 'link',
tokens: [
{
type: 'text',
raw: 'link',
text: 'link',
escaped: false,
},
],
},
],
});
});
it('link title', () => {
expectInlineTokens({
md: '[link](https://example.com "title")',
tokens: [
{
type: 'link',
raw: '[link](https://example.com "title")',
href: 'https://example.com',
title: 'title',
text: 'link',
tokens: [
{
type: 'text',
raw: 'link',
text: 'link',
escaped: false,
},
],
},
],
});
});
it('image', () => {
expectInlineTokens({
md: '',
tokens: [
{
type: 'image',
raw: '',
text: 'image',
href: 'https://example.com/image.png',
title: null,
tokens: [{
type: 'text',
raw: 'image',
text: 'image',
escaped: false,
}],
},
],
});
});
it('image title', () => {
expectInlineTokens({
md: '',
tokens: [
{
type: 'image',
raw: '',
text: 'image',
href: 'https://example.com/image.png',
title: 'title',
tokens: [{
type: 'text',
raw: 'image',
text: 'image',
escaped: false,
}],
},
],
});
});
describe('reflink', () => {
it('reflink', () => {
expectInlineTokens({
md: '[link][]',
links: {
link: { href: 'https://example.com', title: 'title' },
},
tokens: [
{
type: 'link',
raw: '[link][]',
href: 'https://example.com',
title: 'title',
text: 'link',
tokens: [{
type: 'text',
raw: 'link',
text: 'link',
escaped: false,
}],
},
],
});
});
it('nolink', () => {
expectInlineTokens({
md: '[link]',
links: {
link: { href: 'https://example.com', title: 'title' },
},
tokens: [
{
type: 'link',
raw: '[link]',
href: 'https://example.com',
title: 'title',
text: 'link',
tokens: [{
type: 'text',
raw: 'link',
text: 'link',
escaped: false,
}],
},
],
});
});
it('no def', () => {
expectInlineTokens({
md: '[link]',
tokens: [
{
type: 'text',
raw: '[link]',
text: '[link]',
},
],
});
});
});
it('strong', () => {
expectInlineTokens({
md: '**strong**',
tokens: [
{
type: 'strong',
raw: '**strong**',
text: 'strong',
tokens: [
{
type: 'text',
raw: 'strong',
text: 'strong',
escaped: false,
},
],
},
],
});
});
it('em', () => {
expectInlineTokens({
md: '*em*',
tokens: [
{
type: 'em',
raw: '*em*',
text: 'em',
tokens: [
{
type: 'text',
raw: 'em',
text: 'em',
escaped: false,
},
],
},
],
});
});
describe('codespan', () => {
it('code', () => {
expectInlineTokens({
md: '`code`',
tokens: [
{ type: 'codespan', raw: '`code`', text: 'code' },
],
});
});
it('only spaces not stripped', () => {
expectInlineTokens({
md: '` `',
tokens: [
{ type: 'codespan', raw: '` `', text: ' ' },
],
});
});
it('beginning space only not stripped', () => {
expectInlineTokens({
md: '` a`',
tokens: [
{ type: 'codespan', raw: '` a`', text: ' a' },
],
});
});
it('end space only not stripped', () => {
expectInlineTokens({
md: '`a `',
tokens: [
{ type: 'codespan', raw: '`a `', text: 'a ' },
],
});
});
it('begin and end spaces are stripped', () => {
expectInlineTokens({
md: '` a `',
tokens: [
{ type: 'codespan', raw: '` a `', text: 'a' },
],
});
});
it('begin and end newlines are stripped', () => {
expectInlineTokens({
md: '`\na\n`',
tokens: [
{ type: 'codespan', raw: '`\na\n`', text: 'a' },
],
});
});
it('begin and end tabs are not stripped', () => {
expectInlineTokens({
md: '`\ta\t`',
tokens: [
{ type: 'codespan', raw: '`\ta\t`', text: '\ta\t' },
],
});
});
it('begin and end newlines', () => {
expectInlineTokens({
md: '`\na\n`',
tokens: [
{ type: 'codespan', raw: '`\na\n`', text: 'a' },
],
});
});
it('begin and end multiple spaces only one stripped', () => {
expectInlineTokens({
md: '` a `',
tokens: [
{ type: 'codespan', raw: '` a `', text: ' a ' },
],
});
});
it('newline to space', () => {
expectInlineTokens({
md: '`a\nb`',
tokens: [
{ type: 'codespan', raw: '`a\nb`', text: 'a b' },
],
});
});
});
it('br', () => {
expectInlineTokens({
md: 'a\nb',
options: { gfm: true, breaks: true },
tokens: [
{
raw: 'a',
text: 'a',
type: 'text',
escaped: false,
},
{
raw: '\n',
type: 'br',
},
{
raw: 'b',
text: 'b',
type: 'text',
escaped: false,
},
],
});
});
it('del', () => {
expectInlineTokens({
md: '~~del~~',
tokens: [
{
type: 'del',
raw: '~~del~~',
text: 'del',
tokens: [
{
type: 'text',
raw: 'del',
text: 'del',
escaped: false,
},
],
},
],
});
});
describe('url', () => {
it('autolink', () => {
expectInlineTokens({
md: '<https://example.com>',
tokens: [
{
type: 'link',
raw: '<https://example.com>',
text: 'https://example.com',
href: 'https://example.com',
tokens: [
{
type: 'text',
raw: 'https://example.com',
text: 'https://example.com',
},
],
},
],
});
});
it('autolink email', () => {
expectInlineTokens({
md: '<test@example.com>',
options: {},
tokens: [
{
type: 'link',
raw: '<test@example.com>',
text: 'test@example.com',
href: 'mailto:test@example.com',
tokens: [
{
type: 'text',
raw: 'test@example.com',
text: 'test@example.com',
},
],
},
],
});
});
it('url', () => {
expectInlineTokens({
md: 'https://example.com',
tokens: [
{
type: 'link',
raw: 'https://example.com',
text: 'https://example.com',
href: 'https://example.com',
tokens: [
{
type: 'text',
raw: 'https://example.com',
text: 'https://example.com',
},
],
},
],
});
});
it('url email', () => {
expectInlineTokens({
md: 'test@example.com',
options: { gfm: true },
tokens: [
{
type: 'link',
raw: 'test@example.com',
text: 'test@example.com',
href: 'mailto:test@example.com',
tokens: [
{
type: 'text',
raw: 'test@example.com',
text: 'test@example.com',
},
],
},
],
});
});
});
it('text', () => {
expectInlineTokens({
md: 'text',
tokens: [
{
type: 'text',
raw: 'text',
text: 'text',
escaped: false,
},
],
});
});
});
});
describe('multiple instances', () => {
it('should produce identical tokens when multiple Lexer instances lex the same input (issue #3854)', () => {
// Create the first instance and lex — this may mutate shared _defaults.tokenizer.lexer
const lexer1 = new Lexer();
const tokensA = lexer1.lex('Test');
// Create a second instance after the first — previously tokensB.tokens would be empty
// because lexer1's constructor overwrote _defaults.tokenizer.lexer to point to lexer1
const lexer2 = new Lexer();
const tokensB = lexer2.lex('Test');
assert.deepEqual(tokensA, tokensB);
});
it('should produce identical tokens using static lex and instance lex (issue #3854)', () => {
const lexer = new Lexer();
const tokensA = Lexer.lex('Test');
const tokensB = lexer.lex('Test');
assert.deepEqual(tokensA, tokensB);
});
});
});