fix: trim blank lines from block tokens (#3939)

BREAKING CHANGE: trim trailing blank lines from block tokens

Author: UziTech

src/Tokenizer.ts

@@ -4,6 +4,7 @@ import {
   splitCells,
   findClosingBracket,
   expandTabs,
+  trimTrailingBlankLines,
 } from './helpers.ts';
 import type { Rules } from './rules.ts';
 import type { _Lexer } from './Lexer.ts';
@@ -81,14 +82,15 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
   code(src: string): Tokens.Code | undefined {
     const cap = this.rules.block.code.exec(src);
     if (cap) {
-      const text = cap[0].replace(this.rules.other.codeRemoveIndent, '');
+      const raw = this.options.pedantic
+        ? cap[0]
+        : trimTrailingBlankLines(cap[0]);
+      const text = raw.replace(this.rules.other.codeRemoveIndent, '');
       return {
         type: 'code',
-        raw: cap[0],
+        raw,
         codeBlockStyle: 'indented',
-        text: !this.options.pedantic
-          ? rtrim(text, '\n')
-          : text,
+        text,
       };
     }
   }
@@ -481,12 +483,13 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
   html(src: string): Tokens.HTML | undefined {
     const cap = this.rules.block.html.exec(src);
     if (cap) {
+      const raw = trimTrailingBlankLines(cap[0]);
       const token: Tokens.HTML = {
         type: 'html',
         block: true,
-        raw: cap[0],
+        raw,
         pre: cap[1] === 'pre' || cap[1] === 'script' || cap[1] === 'style',
-        text: cap[0],
+        text: raw,
       };
       return token;
     }

src/helpers.ts

@@ -109,6 +109,20 @@ export function rtrim(str: string, c: string, invert?: boolean) {
   return str.slice(0, l - suffLen);
 }
 
+export function trimTrailingBlankLines(str: string) {
+  const lines = str.split('\n');
+  let end = lines.length - 1;
+  while (end >= 0 && !lines[end].trim()) {
+    end--;
+  }
+  if (lines.length - end <= 2) {
+    // we want to keep single trailing blank lines
+    return str;
+  }
+
+  return lines.slice(0, end + 1).join('\n');
+}
+
 export function findClosingBracket(str: string, b: string) {
   if (str.indexOf(b[1]) === -1) {
     return -1;

test/unit/Lexer.test.js

@@ -59,6 +59,22 @@ describe('Lexer', () => {
       });
     });
 
+    it('indented code with trailing blank lines', () => {
+      expectTokens({
+        md: '    code\n     \n\t\t\n \t\n\nnext',
+        tokens: [
+          { type: 'code', raw: '    code', text: 'code', codeBlockStyle: 'indented' },
+          { type: 'space', raw: '\n     \n\t\t\n \t\n\n' },
+          {
+            type: 'paragraph',
+            raw: 'next',
+            text: 'next',
+            tokens: [{ type: 'text', raw: 'next', text: 'next', escaped: false }],
+          },
+        ],
+      });
+    });
+
     it('fenced code', () => {
       expectTokens({
         md: '```\ncode\n```',
@@ -1426,6 +1442,31 @@ paragraph
         ],
       });
     });
+
+    it('html with trailing blank lines', () => {
+      expectTokens({
+        md: '<div>\n  html\n</div>\n \n\t\n\nnext',
+        tokens: [
+          {
+            type: 'html',
+            raw: '<div>\n  html\n</div>',
+            pre: false,
+            block: true,
+            text: '<div>\n  html\n</div>',
+          },
+          {
+            type: 'space',
+            raw: '\n \n\t\n\n',
+          },
+          {
+            type: 'paragraph',
+            raw: 'next',
+            text: 'next',
+            tokens: [{ type: 'text', raw: 'next', text: 'next', escaped: false }],
+          },
+        ],
+      });
+    });
   });
 
   describe('def', () => {

test/unit/marked.test.js

@@ -1027,6 +1027,7 @@ br
         ['text', 'list'],
         ['space', ''],
         ['html', '<div>html</div>'],
+        ['space', ''],
         ['paragraph', '[link](https://example.com)'],
         ['link', '[link](https://example.com)'],
         ['text', 'link'],