perf: decode ASCII varchar values without iconv

src/token/nbcrow-token-parser.ts

@@ -4,9 +4,8 @@ import Parser from './stream-parser';
 import { type ColumnMetadata } from './colmetadata-token-parser';
 
 import { NBCRowToken } from './token';
-import * as iconv from 'iconv-lite';
 
-import { isPLPStream, readPLPStream, readValue } from '../value-parser';
+import { decodeChars, isPLPStream, readPLPStream, readValue } from '../value-parser';
 import { NotEnoughDataError } from './helpers';
 
 interface Column {
@@ -56,7 +55,7 @@ async function nbcRowParser(parser: Parser): Promise<NBCRowToken> {
         } else if (metadata.type.name === 'NVarChar' || metadata.type.name === 'Xml') {
           columns.push({ value: Buffer.concat(chunks).toString('ucs2'), metadata });
         } else if (metadata.type.name === 'VarChar') {
-          columns.push({ value: iconv.decode(Buffer.concat(chunks), metadata.collation?.codepage ?? 'utf8'), metadata });
+          columns.push({ value: decodeChars(Buffer.concat(chunks), metadata.collation?.codepage ?? 'utf-8'), metadata });
         } else if (metadata.type.name === 'VarBinary' || metadata.type.name === 'UDT') {
           columns.push({ value: Buffer.concat(chunks), metadata });
         }

src/token/returnvalue-token-parser.ts

@@ -5,9 +5,8 @@ import Parser from './stream-parser';
 import { ReturnValueToken } from './token';
 
 import { readMetadata } from '../metadata-parser';
-import { isPLPStream, readPLPStream, readValue } from '../value-parser';
+import { decodeChars, isPLPStream, readPLPStream, readValue } from '../value-parser';
 import { NotEnoughDataError, readBVarChar, readUInt16LE, readUInt8 } from './helpers';
-import * as iconv from 'iconv-lite';
 
 async function returnParser(parser: Parser): Promise<ReturnValueToken> {
   let paramName;
@@ -54,7 +53,7 @@ async function returnParser(parser: Parser): Promise<ReturnValueToken> {
       } else if (metadata.type.name === 'NVarChar' || metadata.type.name === 'Xml') {
         value = Buffer.concat(chunks).toString('ucs2');
       } else if (metadata.type.name === 'VarChar') {
-        value = iconv.decode(Buffer.concat(chunks), metadata.collation?.codepage ?? 'utf8');
+        value = decodeChars(Buffer.concat(chunks), metadata.collation?.codepage ?? 'utf-8');
       } else if (metadata.type.name === 'VarBinary' || metadata.type.name === 'UDT') {
         value = Buffer.concat(chunks);
       }

src/token/row-token-parser.ts

@@ -4,9 +4,8 @@ import Parser from './stream-parser';
 import { type ColumnMetadata } from './colmetadata-token-parser';
 
 import { RowToken } from './token';
-import * as iconv from 'iconv-lite';
 
-import { isPLPStream, readPLPStream, readValue } from '../value-parser';
+import { decodeChars, isPLPStream, readPLPStream, readValue } from '../value-parser';
 import { NotEnoughDataError } from './helpers';
 
 interface Column {
@@ -27,7 +26,7 @@ async function rowParser(parser: Parser): Promise<RowToken> {
         } else if (metadata.type.name === 'NVarChar' || metadata.type.name === 'Xml') {
           columns.push({ value: Buffer.concat(chunks).toString('ucs2'), metadata });
         } else if (metadata.type.name === 'VarChar') {
-          columns.push({ value: iconv.decode(Buffer.concat(chunks), metadata.collation?.codepage ?? 'utf8'), metadata });
+          columns.push({ value: decodeChars(Buffer.concat(chunks), metadata.collation?.codepage ?? 'utf-8'), metadata });
         } else if (metadata.type.name === 'VarBinary' || metadata.type.name === 'UDT') {
           columns.push({ value: Buffer.concat(chunks), metadata });
         }

src/value-parser.ts

@@ -3,6 +3,7 @@ import { type Metadata, readCollation } from './metadata-parser';
 import { TYPE } from './data-type';
 
 import iconv from 'iconv-lite';
+import { isAscii } from 'node:buffer';
 import { sprintf } from 'sprintf-js';
 import { bufferToLowerCaseGuid, bufferToUpperCaseGuid } from './guid-parser';
 import { NotEnoughDataError, Result, readBigInt64LE, readDoubleLE, readFloatLE, readInt16LE, readInt32LE, readUInt16LE, readUInt32LE, readUInt8, readUInt24LE, readUInt40LE, readUNumeric64LE, readUNumeric96LE, readUNumeric128LE } from './token/helpers';
@@ -13,7 +14,7 @@ const THREE_AND_A_THIRD = 3 + (1 / 3);
 const MONEY_DIVISOR = 10000;
 const PLP_NULL = 0xFFFFFFFFFFFFFFFFn;
 const UNKNOWN_PLP_LEN = 0xFFFFFFFFFFFFFFFEn;
-const DEFAULT_ENCODING = 'utf8';
+const DEFAULT_ENCODING = 'utf-8';
 
 function readTinyInt(buf: Buffer, offset: number): Result<number> {
   return readUInt8(buf, offset);
@@ -568,12 +569,63 @@ function readBinary(buf: Buffer, offset: number, dataLength: number): Result<Buf
   return new Result(buf.slice(offset, offset + dataLength), offset + dataLength);
 }
 
+// Codepages that decode the 7-bit ASCII range identically to ASCII,
+// allowing a fast path in `readChars`. This covers every codepage that a
+// `Collation` can produce (see `codepageByLanguageId` / `codepageBySortId`),
+// plus `DEFAULT_ENCODING`.
+//
+// A codepage missing from this list only loses the fast path - decoding
+// still works correctly via `iconv`.
+const asciiCompatibleCodepages = new Set([
+  'CP437', 'CP850', 'CP874',
+  'CP932', 'CP936', 'CP949', 'CP950',
+  'CP1250', 'CP1251', 'CP1252', 'CP1253', 'CP1254', 'CP1255', 'CP1256', 'CP1257', 'CP1258',
+  'utf-8'
+]);
+
 function readChars(buf: Buffer, offset: number, dataLength: number, codepage: string): Result<string> {
   if (buf.length < offset + dataLength) {
     throw new NotEnoughDataError(offset + dataLength);
   }
 
-  return new Result(iconv.decode(buf.slice(offset, offset + dataLength), codepage ?? DEFAULT_ENCODING), offset + dataLength);
+  const data = buf.slice(offset, offset + dataLength);
+
+  // Fast path: pure ASCII data in an ASCII compatible codepage can be
+  // decoded natively, skipping the (much slower) `iconv` decoding.
+  if (asciiCompatibleCodepages.has(codepage ?? DEFAULT_ENCODING) && isAscii(data)) {
+    return new Result(data.toString('latin1'), offset + dataLength);
+  }
+
+  return new Result(decodeChars(data, codepage ?? DEFAULT_ENCODING), offset + dataLength);
+}
+
+const decodersByCodepage = new Map<string, ReturnType<typeof iconv.getDecoder>>();
+
+// Decodes a complete value, treating the bytes as pure character data: a
+// leading byte order mark is *not* stripped, matching how `nvarchar` values
+// and other SQL Server drivers handle it. (`iconv.decode` would strip it.)
+//
+// UTF-8 values are decoded natively. Everything else is decoded via
+// `iconv`, reusing one decoder per codepage instead of letting `iconv`
+// create a fresh one for every value. This is safe because a decoder that
+// has fully consumed its input via `write` + `end` is back in its initial
+// state, and `stripBOM: false` avoids iconv's BOM wrapper, which is the
+// only decoder layer that carries state from one value to the next.
+function decodeChars(data: Buffer, codepage: string): string {
+  if (codepage === 'utf-8') {
+    return data.toString('utf8');
+  }
+
+  let decoder = decodersByCodepage.get(codepage);
+  if (decoder === undefined) {
+    decoder = iconv.getDecoder(codepage, { stripBOM: false });
+    decodersByCodepage.set(codepage, decoder);
+  }
+
+  const result = decoder.write(data);
+  const trailer = decoder.end();
+
+  return trailer ? result + trailer : result;
 }
 
 function readNChars(buf: Buffer, offset: number, dataLength: number): Result<string> {
@@ -787,5 +839,6 @@ function readDateTimeOffset(buf: Buffer, offset: number, dataLength: number, sca
 module.exports.readValue = readValue;
 module.exports.isPLPStream = isPLPStream;
 module.exports.readPLPStream = readPLPStream;
+module.exports.decodeChars = decodeChars;
 
-export { readValue, isPLPStream, readPLPStream };
+export { readValue, isPLPStream, readPLPStream, decodeChars };

test/unit/token/row-token-parser-test.ts

@@ -13,7 +13,7 @@ import { type ColumnMetadata } from '../../../src/token/colmetadata-token-parser
 import { typeByName as dataTypeByName } from '../../../src/data-type';
 import WritableTrackingBuffer from '../../../src/tracking-buffer/writable-tracking-buffer';
 import Debug from '../../../src/debug';
-import { Collation } from '../../../src/collation';
+import { Collation, Flags } from '../../../src/collation';
 
 const options = {
   useUTC: false,
@@ -486,6 +486,46 @@ describe('Row Token Parser', function() {
     assert.isTrue((await parser.next()).done);
   });
 
+  it('should parse a varchar(max) value in a UTF-8 collation', async function() {
+    const debug = new Debug();
+    const colMetadata: ColumnMetadata[] = [
+      {
+        colName: 'col0',
+        userType: 0,
+        flags: 0,
+        precision: undefined,
+        scale: undefined,
+        dataLength: 65535,
+        schema: undefined,
+        udtInfo: undefined,
+        type: dataTypeByName.VarChar,
+        collation: new Collation(1033, Flags.UTF8, 0, 0)
+      }
+    ];
+
+    // A value starting with a byte order mark - it is part of the data and
+    // must be preserved.
+    const value = Buffer.concat([Buffer.from([0xEF, 0xBB, 0xBF]), Buffer.from('h\u00e9llo \u6771\u4eac')]);
+
+    const buffer = new WritableTrackingBuffer(0, 'ascii');
+    buffer.writeUInt8(0xd1);
+    buffer.writeUInt64LE(value.length);
+    buffer.writeUInt32LE(value.length);
+    buffer.writeBuffer(value);
+    buffer.writeUInt32LE(0);
+
+    const parser = Parser.parseTokens([buffer.data], debug, options, colMetadata);
+    const result = await parser.next();
+    assert.isFalse(result.done);
+    const token = result.value;
+
+    assert.instanceOf(token, RowToken);
+    assert.strictEqual(token.columns.length, 1);
+    assert.strictEqual(token.columns[0].value, '\uFEFFh\u00e9llo \u6771\u4eac');
+    assert.strictEqual(token.columns[0].metadata, colMetadata[0]);
+    assert.isTrue((await parser.next()).done);
+  });
+
   it('should parse varcharMaxNull', async function() {
     const debug = new Debug();
     const colMetadata: ColumnMetadata[] = [

test/unit/value-parser-test.ts

@@ -4,6 +4,8 @@ import { readValue } from '../../src/value-parser';
 import { type Metadata } from '../../src/metadata-parser';
 import { type ParserOptions } from '../../src/token/stream-parser';
 import { type DataType, typeByName as dataTypeByName } from '../../src/data-type';
+import { codepageByLanguageId, codepageBySortId } from '../../src/collation';
+import iconv from 'iconv-lite';
 
 const utcOptions = { useUTC: true } as ParserOptions;
 const localOptions = { useUTC: false } as ParserOptions;
@@ -332,4 +334,155 @@ describe('readValue', function() {
       assert.strictEqual(value.getTime(), new Date(1900, 0, 3, 0, 0, 45).getTime());
     });
   });
+
+  describe('for `varchar` and `char` values', function() {
+    function buildCharMetadata(type: DataType, codepage: string): Metadata {
+      const metadata = buildMetadata(type);
+      metadata.collation = { codepage } as unknown as Metadata['collation'];
+      return metadata;
+    }
+
+    function buildCharBuffer(data: Buffer): Buffer {
+      const buf = Buffer.alloc(2 + data.length);
+      buf.writeUInt16LE(data.length, 0);
+      data.copy(buf, 2);
+      return buf;
+    }
+
+    it('should parse ASCII values in different codepages', function() {
+      const data = Buffer.from('user12345@example.com, The quick brown fox!', 'latin1');
+
+      for (const codepage of ['CP1252', 'CP437', 'CP850', 'CP932', 'CP936', 'utf8']) {
+        const buf = buildCharBuffer(data);
+        const result = readValue(buf, 0, buildCharMetadata(dataTypeByName.VarChar, codepage), utcOptions);
+
+        assert.strictEqual(result.value, 'user12345@example.com, The quick brown fox!', codepage);
+        assert.strictEqual(result.offset, buf.length, codepage);
+      }
+    });
+
+    it('should parse values containing non-ASCII single-byte characters', function() {
+      // "café" in CP1252: 0xE9 is "é"
+      const buf = buildCharBuffer(Buffer.from([0x63, 0x61, 0x66, 0xE9]));
+      const result = readValue(buf, 0, buildCharMetadata(dataTypeByName.VarChar, 'CP1252'), utcOptions);
+
+      assert.strictEqual(result.value, 'café');
+      assert.strictEqual(result.offset, buf.length);
+    });
+
+    it('should parse values containing multi-byte characters', function() {
+      // "あA" in CP932 (Shift-JIS): 0x82 0xA0 is "あ"
+      const buf = buildCharBuffer(Buffer.from([0x82, 0xA0, 0x41]));
+      const result = readValue(buf, 0, buildCharMetadata(dataTypeByName.VarChar, 'CP932'), utcOptions);
+
+      assert.strictEqual(result.value, 'あA');
+      assert.strictEqual(result.offset, buf.length);
+    });
+
+    it('should decode ASCII bytes natively for every codepage a collation can produce', function() {
+      // The native fast path in `readChars` is gated on a hardcoded list of
+      // ASCII compatible codepages. Verify that every codepage that can come
+      // out of the collation tables (plus the `utf8` fallback encoding)
+      // actually decodes the 7-bit ASCII range identically to ASCII, and
+      // that `readValue` takes the same result either way.
+      const codepages = new Set([...Object.values(codepageByLanguageId), ...Object.values(codepageBySortId), 'utf-8', 'utf8']);
+
+      const probe = Buffer.alloc(128);
+      for (let i = 0; i < 128; i++) {
+        probe[i] = i;
+      }
+
+      for (const codepage of codepages) {
+        assert.strictEqual(iconv.decode(probe, codepage), probe.toString('latin1'), codepage);
+
+        const buf = buildCharBuffer(Buffer.from('plain ASCII value', 'latin1'));
+        const result = readValue(buf, 0, buildCharMetadata(dataTypeByName.VarChar, codepage), utcOptions);
+        assert.strictEqual(result.value, 'plain ASCII value', codepage);
+      }
+    });
+
+    it('should parse ASCII values in codepages that do not decode ASCII bytes as ASCII', function() {
+      // In UTF-16BE, the ASCII bytes "ab" decode to a single character (U+6162).
+      const buf = buildCharBuffer(Buffer.from('ab', 'latin1'));
+      const result = readValue(buf, 0, buildCharMetadata(dataTypeByName.VarChar, 'utf-16be'), utcOptions);
+
+      assert.strictEqual(result.value, '慢');
+    });
+
+    it('should parse empty values', function() {
+      const buf = buildCharBuffer(Buffer.alloc(0));
+      const result = readValue(buf, 0, buildCharMetadata(dataTypeByName.VarChar, 'CP1252'), utcOptions);
+
+      assert.strictEqual(result.value, '');
+      assert.strictEqual(result.offset, buf.length);
+    });
+
+    it('should parse `NULL` values', function() {
+      const buf = Buffer.alloc(2);
+      buf.writeUInt16LE(0xFFFF, 0);
+
+      const result = readValue(buf, 0, buildCharMetadata(dataTypeByName.VarChar, 'CP1252'), utcOptions);
+
+      assert.isNull(result.value);
+      assert.strictEqual(result.offset, buf.length);
+    });
+
+    it('should parse `char` values', function() {
+      const buf = buildCharBuffer(Buffer.from('fixed     ', 'latin1'));
+      const result = readValue(buf, 0, buildCharMetadata(dataTypeByName.Char, 'CP1252'), utcOptions);
+
+      assert.strictEqual(result.value, 'fixed     ');
+    });
+
+    it('should parse UTF-8 values', function() {
+      const data = Buffer.from('Müller – 東京 🚀');
+      const result = readValue(buildCharBuffer(data), 0, buildCharMetadata(dataTypeByName.VarChar, 'utf-8'), utcOptions);
+
+      assert.strictEqual(result.value, 'Müller – 東京 🚀');
+    });
+
+    it('should preserve a leading byte order mark', function() {
+      // U+FEFF at the start of a value is part of the stored data, not an
+      // encoding marker - it must be returned, matching how `nvarchar`
+      // values are handled.
+      const data = Buffer.concat([Buffer.from([0xEF, 0xBB, 0xBF]), Buffer.from('héllo')]);
+
+      for (let i = 0; i < 3; i++) {
+        const result = readValue(buildCharBuffer(data), 0, buildCharMetadata(dataTypeByName.VarChar, 'utf-8'), utcOptions);
+        assert.strictEqual(result.value, '\uFEFFhéllo', `iteration ${i}`);
+      }
+    });
+
+    it('should parse malformed UTF-8 values like iconv does', function() {
+      const cases = [
+        Buffer.from([0x61, 0xE3, 0x81]), // "a" + truncated 3-byte sequence
+        Buffer.from([0x61, 0x80, 0x62]), // lone continuation byte
+        Buffer.from([0xF0, 0x9F, 0x62]), // truncated 4-byte sequence
+        Buffer.from([0xC0, 0xAF]) // overlong encoding
+      ];
+
+      for (const data of cases) {
+        const result = readValue(buildCharBuffer(data), 0, buildCharMetadata(dataTypeByName.VarChar, 'utf-8'), utcOptions);
+        assert.strictEqual(result.value, iconv.decode(data, 'utf-8'), data.toString('hex'));
+      }
+    });
+
+    it('should parse values independently of previously parsed values', function() {
+      // A value that ends in a truncated CP932 multi-byte sequence...
+      const truncated = Buffer.concat([iconv.encode('テスト', 'CP932'), Buffer.from([0x82])]);
+      const first = readValue(buildCharBuffer(truncated), 0, buildCharMetadata(dataTypeByName.VarChar, 'CP932'), utcOptions);
+      assert.strictEqual(first.value, iconv.decode(truncated, 'CP932'));
+
+      // ...must not leak into the decoding of the next value.
+      const second = readValue(buildCharBuffer(Buffer.from([0x82, 0xA0, 0x41])), 0, buildCharMetadata(dataTypeByName.VarChar, 'CP932'), utcOptions);
+      assert.strictEqual(second.value, 'あA');
+    });
+
+    it('should parse values that are ASCII except for the final byte', function() {
+      const buf = buildCharBuffer(Buffer.from([0x61, 0x62, 0x63, 0xFC])); // "abcü" in CP1252
+      const result = readValue(buf, 0, buildCharMetadata(dataTypeByName.VarChar, 'CP1252'), utcOptions);
+
+      assert.strictEqual(result.value, 'abcü');
+    });
+  });
 });