diff --git a/src/token/nbcrow-token-parser.ts b/src/token/nbcrow-token-parser.ts index 0b8d07468..e7ebd5a7a 100644 --- a/src/token/nbcrow-token-parser.ts +++ b/src/token/nbcrow-token-parser.ts @@ -4,9 +4,8 @@ import Parser from './stream-parser'; import { type ColumnMetadata } from './colmetadata-token-parser'; import { NBCRowToken } from './token'; -import * as iconv from 'iconv-lite'; -import { isPLPStream, readPLPStream, readValue } from '../value-parser'; +import { decodeChars, isPLPStream, readPLPStream, readValue } from '../value-parser'; import { NotEnoughDataError } from './helpers'; interface Column { @@ -56,7 +55,7 @@ async function nbcRowParser(parser: Parser): Promise { } else if (metadata.type.name === 'NVarChar' || metadata.type.name === 'Xml') { columns.push({ value: Buffer.concat(chunks).toString('ucs2'), metadata }); } else if (metadata.type.name === 'VarChar') { - columns.push({ value: iconv.decode(Buffer.concat(chunks), metadata.collation?.codepage ?? 'utf8'), metadata }); + columns.push({ value: decodeChars(Buffer.concat(chunks), metadata.collation?.codepage ?? 'utf-8'), metadata }); } else if (metadata.type.name === 'VarBinary' || metadata.type.name === 'UDT') { columns.push({ value: Buffer.concat(chunks), metadata }); } diff --git a/src/token/returnvalue-token-parser.ts b/src/token/returnvalue-token-parser.ts index bcfa14ced..d38e07abe 100644 --- a/src/token/returnvalue-token-parser.ts +++ b/src/token/returnvalue-token-parser.ts @@ -5,9 +5,8 @@ import Parser from './stream-parser'; import { ReturnValueToken } from './token'; import { readMetadata } from '../metadata-parser'; -import { isPLPStream, readPLPStream, readValue } from '../value-parser'; +import { decodeChars, isPLPStream, readPLPStream, readValue } from '../value-parser'; import { NotEnoughDataError, readBVarChar, readUInt16LE, readUInt8 } from './helpers'; -import * as iconv from 'iconv-lite'; async function returnParser(parser: Parser): Promise { let paramName; @@ -54,7 +53,7 @@ async function returnParser(parser: Parser): Promise { } else if (metadata.type.name === 'NVarChar' || metadata.type.name === 'Xml') { value = Buffer.concat(chunks).toString('ucs2'); } else if (metadata.type.name === 'VarChar') { - value = iconv.decode(Buffer.concat(chunks), metadata.collation?.codepage ?? 'utf8'); + value = decodeChars(Buffer.concat(chunks), metadata.collation?.codepage ?? 'utf-8'); } else if (metadata.type.name === 'VarBinary' || metadata.type.name === 'UDT') { value = Buffer.concat(chunks); } diff --git a/src/token/row-token-parser.ts b/src/token/row-token-parser.ts index f394dbf97..ef7107399 100644 --- a/src/token/row-token-parser.ts +++ b/src/token/row-token-parser.ts @@ -4,9 +4,8 @@ import Parser from './stream-parser'; import { type ColumnMetadata } from './colmetadata-token-parser'; import { RowToken } from './token'; -import * as iconv from 'iconv-lite'; -import { isPLPStream, readPLPStream, readValue } from '../value-parser'; +import { decodeChars, isPLPStream, readPLPStream, readValue } from '../value-parser'; import { NotEnoughDataError } from './helpers'; interface Column { @@ -27,7 +26,7 @@ async function rowParser(parser: Parser): Promise { } else if (metadata.type.name === 'NVarChar' || metadata.type.name === 'Xml') { columns.push({ value: Buffer.concat(chunks).toString('ucs2'), metadata }); } else if (metadata.type.name === 'VarChar') { - columns.push({ value: iconv.decode(Buffer.concat(chunks), metadata.collation?.codepage ?? 'utf8'), metadata }); + columns.push({ value: decodeChars(Buffer.concat(chunks), metadata.collation?.codepage ?? 'utf-8'), metadata }); } else if (metadata.type.name === 'VarBinary' || metadata.type.name === 'UDT') { columns.push({ value: Buffer.concat(chunks), metadata }); } diff --git a/src/value-parser.ts b/src/value-parser.ts index 32ba6d1e7..cb7dba97b 100644 --- a/src/value-parser.ts +++ b/src/value-parser.ts @@ -3,6 +3,7 @@ import { type Metadata, readCollation } from './metadata-parser'; import { TYPE } from './data-type'; import iconv from 'iconv-lite'; +import { isAscii } from 'node:buffer'; import { sprintf } from 'sprintf-js'; import { bufferToLowerCaseGuid, bufferToUpperCaseGuid } from './guid-parser'; import { NotEnoughDataError, Result, readBigInt64LE, readDoubleLE, readFloatLE, readInt16LE, readInt32LE, readUInt16LE, readUInt32LE, readUInt8, readUInt24LE, readUInt40LE, readUNumeric64LE, readUNumeric96LE, readUNumeric128LE } from './token/helpers'; @@ -13,7 +14,7 @@ const THREE_AND_A_THIRD = 3 + (1 / 3); const MONEY_DIVISOR = 10000; const PLP_NULL = 0xFFFFFFFFFFFFFFFFn; const UNKNOWN_PLP_LEN = 0xFFFFFFFFFFFFFFFEn; -const DEFAULT_ENCODING = 'utf8'; +const DEFAULT_ENCODING = 'utf-8'; function readTinyInt(buf: Buffer, offset: number): Result { return readUInt8(buf, offset); @@ -568,12 +569,63 @@ function readBinary(buf: Buffer, offset: number, dataLength: number): Result { if (buf.length < offset + dataLength) { throw new NotEnoughDataError(offset + dataLength); } - return new Result(iconv.decode(buf.slice(offset, offset + dataLength), codepage ?? DEFAULT_ENCODING), offset + dataLength); + const data = buf.slice(offset, offset + dataLength); + + // Fast path: pure ASCII data in an ASCII compatible codepage can be + // decoded natively, skipping the (much slower) `iconv` decoding. + if (asciiCompatibleCodepages.has(codepage ?? DEFAULT_ENCODING) && isAscii(data)) { + return new Result(data.toString('latin1'), offset + dataLength); + } + + return new Result(decodeChars(data, codepage ?? DEFAULT_ENCODING), offset + dataLength); +} + +const decodersByCodepage = new Map>(); + +// Decodes a complete value, treating the bytes as pure character data: a +// leading byte order mark is *not* stripped, matching how `nvarchar` values +// and other SQL Server drivers handle it. (`iconv.decode` would strip it.) +// +// UTF-8 values are decoded natively. Everything else is decoded via +// `iconv`, reusing one decoder per codepage instead of letting `iconv` +// create a fresh one for every value. This is safe because a decoder that +// has fully consumed its input via `write` + `end` is back in its initial +// state, and `stripBOM: false` avoids iconv's BOM wrapper, which is the +// only decoder layer that carries state from one value to the next. +function decodeChars(data: Buffer, codepage: string): string { + if (codepage === 'utf-8') { + return data.toString('utf8'); + } + + let decoder = decodersByCodepage.get(codepage); + if (decoder === undefined) { + decoder = iconv.getDecoder(codepage, { stripBOM: false }); + decodersByCodepage.set(codepage, decoder); + } + + const result = decoder.write(data); + const trailer = decoder.end(); + + return trailer ? result + trailer : result; } function readNChars(buf: Buffer, offset: number, dataLength: number): Result { @@ -787,5 +839,6 @@ function readDateTimeOffset(buf: Buffer, offset: number, dataLength: number, sca module.exports.readValue = readValue; module.exports.isPLPStream = isPLPStream; module.exports.readPLPStream = readPLPStream; +module.exports.decodeChars = decodeChars; -export { readValue, isPLPStream, readPLPStream }; +export { readValue, isPLPStream, readPLPStream, decodeChars }; diff --git a/test/unit/token/row-token-parser-test.ts b/test/unit/token/row-token-parser-test.ts index f45c0e4b3..7056816c4 100644 --- a/test/unit/token/row-token-parser-test.ts +++ b/test/unit/token/row-token-parser-test.ts @@ -13,7 +13,7 @@ import { type ColumnMetadata } from '../../../src/token/colmetadata-token-parser import { typeByName as dataTypeByName } from '../../../src/data-type'; import WritableTrackingBuffer from '../../../src/tracking-buffer/writable-tracking-buffer'; import Debug from '../../../src/debug'; -import { Collation } from '../../../src/collation'; +import { Collation, Flags } from '../../../src/collation'; const options = { useUTC: false, @@ -486,6 +486,46 @@ describe('Row Token Parser', function() { assert.isTrue((await parser.next()).done); }); + it('should parse a varchar(max) value in a UTF-8 collation', async function() { + const debug = new Debug(); + const colMetadata: ColumnMetadata[] = [ + { + colName: 'col0', + userType: 0, + flags: 0, + precision: undefined, + scale: undefined, + dataLength: 65535, + schema: undefined, + udtInfo: undefined, + type: dataTypeByName.VarChar, + collation: new Collation(1033, Flags.UTF8, 0, 0) + } + ]; + + // A value starting with a byte order mark - it is part of the data and + // must be preserved. + const value = Buffer.concat([Buffer.from([0xEF, 0xBB, 0xBF]), Buffer.from('h\u00e9llo \u6771\u4eac')]); + + const buffer = new WritableTrackingBuffer(0, 'ascii'); + buffer.writeUInt8(0xd1); + buffer.writeUInt64LE(value.length); + buffer.writeUInt32LE(value.length); + buffer.writeBuffer(value); + buffer.writeUInt32LE(0); + + const parser = Parser.parseTokens([buffer.data], debug, options, colMetadata); + const result = await parser.next(); + assert.isFalse(result.done); + const token = result.value; + + assert.instanceOf(token, RowToken); + assert.strictEqual(token.columns.length, 1); + assert.strictEqual(token.columns[0].value, '\uFEFFh\u00e9llo \u6771\u4eac'); + assert.strictEqual(token.columns[0].metadata, colMetadata[0]); + assert.isTrue((await parser.next()).done); + }); + it('should parse varcharMaxNull', async function() { const debug = new Debug(); const colMetadata: ColumnMetadata[] = [ diff --git a/test/unit/value-parser-test.ts b/test/unit/value-parser-test.ts index 32c1463e4..92b94ffee 100644 --- a/test/unit/value-parser-test.ts +++ b/test/unit/value-parser-test.ts @@ -4,6 +4,8 @@ import { readValue } from '../../src/value-parser'; import { type Metadata } from '../../src/metadata-parser'; import { type ParserOptions } from '../../src/token/stream-parser'; import { type DataType, typeByName as dataTypeByName } from '../../src/data-type'; +import { codepageByLanguageId, codepageBySortId } from '../../src/collation'; +import iconv from 'iconv-lite'; const utcOptions = { useUTC: true } as ParserOptions; const localOptions = { useUTC: false } as ParserOptions; @@ -332,4 +334,155 @@ describe('readValue', function() { assert.strictEqual(value.getTime(), new Date(1900, 0, 3, 0, 0, 45).getTime()); }); }); + + describe('for `varchar` and `char` values', function() { + function buildCharMetadata(type: DataType, codepage: string): Metadata { + const metadata = buildMetadata(type); + metadata.collation = { codepage } as unknown as Metadata['collation']; + return metadata; + } + + function buildCharBuffer(data: Buffer): Buffer { + const buf = Buffer.alloc(2 + data.length); + buf.writeUInt16LE(data.length, 0); + data.copy(buf, 2); + return buf; + } + + it('should parse ASCII values in different codepages', function() { + const data = Buffer.from('user12345@example.com, The quick brown fox!', 'latin1'); + + for (const codepage of ['CP1252', 'CP437', 'CP850', 'CP932', 'CP936', 'utf8']) { + const buf = buildCharBuffer(data); + const result = readValue(buf, 0, buildCharMetadata(dataTypeByName.VarChar, codepage), utcOptions); + + assert.strictEqual(result.value, 'user12345@example.com, The quick brown fox!', codepage); + assert.strictEqual(result.offset, buf.length, codepage); + } + }); + + it('should parse values containing non-ASCII single-byte characters', function() { + // "café" in CP1252: 0xE9 is "é" + const buf = buildCharBuffer(Buffer.from([0x63, 0x61, 0x66, 0xE9])); + const result = readValue(buf, 0, buildCharMetadata(dataTypeByName.VarChar, 'CP1252'), utcOptions); + + assert.strictEqual(result.value, 'café'); + assert.strictEqual(result.offset, buf.length); + }); + + it('should parse values containing multi-byte characters', function() { + // "あA" in CP932 (Shift-JIS): 0x82 0xA0 is "あ" + const buf = buildCharBuffer(Buffer.from([0x82, 0xA0, 0x41])); + const result = readValue(buf, 0, buildCharMetadata(dataTypeByName.VarChar, 'CP932'), utcOptions); + + assert.strictEqual(result.value, 'あA'); + assert.strictEqual(result.offset, buf.length); + }); + + it('should decode ASCII bytes natively for every codepage a collation can produce', function() { + // The native fast path in `readChars` is gated on a hardcoded list of + // ASCII compatible codepages. Verify that every codepage that can come + // out of the collation tables (plus the `utf8` fallback encoding) + // actually decodes the 7-bit ASCII range identically to ASCII, and + // that `readValue` takes the same result either way. + const codepages = new Set([...Object.values(codepageByLanguageId), ...Object.values(codepageBySortId), 'utf-8', 'utf8']); + + const probe = Buffer.alloc(128); + for (let i = 0; i < 128; i++) { + probe[i] = i; + } + + for (const codepage of codepages) { + assert.strictEqual(iconv.decode(probe, codepage), probe.toString('latin1'), codepage); + + const buf = buildCharBuffer(Buffer.from('plain ASCII value', 'latin1')); + const result = readValue(buf, 0, buildCharMetadata(dataTypeByName.VarChar, codepage), utcOptions); + assert.strictEqual(result.value, 'plain ASCII value', codepage); + } + }); + + it('should parse ASCII values in codepages that do not decode ASCII bytes as ASCII', function() { + // In UTF-16BE, the ASCII bytes "ab" decode to a single character (U+6162). + const buf = buildCharBuffer(Buffer.from('ab', 'latin1')); + const result = readValue(buf, 0, buildCharMetadata(dataTypeByName.VarChar, 'utf-16be'), utcOptions); + + assert.strictEqual(result.value, '慢'); + }); + + it('should parse empty values', function() { + const buf = buildCharBuffer(Buffer.alloc(0)); + const result = readValue(buf, 0, buildCharMetadata(dataTypeByName.VarChar, 'CP1252'), utcOptions); + + assert.strictEqual(result.value, ''); + assert.strictEqual(result.offset, buf.length); + }); + + it('should parse `NULL` values', function() { + const buf = Buffer.alloc(2); + buf.writeUInt16LE(0xFFFF, 0); + + const result = readValue(buf, 0, buildCharMetadata(dataTypeByName.VarChar, 'CP1252'), utcOptions); + + assert.isNull(result.value); + assert.strictEqual(result.offset, buf.length); + }); + + it('should parse `char` values', function() { + const buf = buildCharBuffer(Buffer.from('fixed ', 'latin1')); + const result = readValue(buf, 0, buildCharMetadata(dataTypeByName.Char, 'CP1252'), utcOptions); + + assert.strictEqual(result.value, 'fixed '); + }); + + it('should parse UTF-8 values', function() { + const data = Buffer.from('Müller – 東京 🚀'); + const result = readValue(buildCharBuffer(data), 0, buildCharMetadata(dataTypeByName.VarChar, 'utf-8'), utcOptions); + + assert.strictEqual(result.value, 'Müller – 東京 🚀'); + }); + + it('should preserve a leading byte order mark', function() { + // U+FEFF at the start of a value is part of the stored data, not an + // encoding marker - it must be returned, matching how `nvarchar` + // values are handled. + const data = Buffer.concat([Buffer.from([0xEF, 0xBB, 0xBF]), Buffer.from('héllo')]); + + for (let i = 0; i < 3; i++) { + const result = readValue(buildCharBuffer(data), 0, buildCharMetadata(dataTypeByName.VarChar, 'utf-8'), utcOptions); + assert.strictEqual(result.value, '\uFEFFhéllo', `iteration ${i}`); + } + }); + + it('should parse malformed UTF-8 values like iconv does', function() { + const cases = [ + Buffer.from([0x61, 0xE3, 0x81]), // "a" + truncated 3-byte sequence + Buffer.from([0x61, 0x80, 0x62]), // lone continuation byte + Buffer.from([0xF0, 0x9F, 0x62]), // truncated 4-byte sequence + Buffer.from([0xC0, 0xAF]) // overlong encoding + ]; + + for (const data of cases) { + const result = readValue(buildCharBuffer(data), 0, buildCharMetadata(dataTypeByName.VarChar, 'utf-8'), utcOptions); + assert.strictEqual(result.value, iconv.decode(data, 'utf-8'), data.toString('hex')); + } + }); + + it('should parse values independently of previously parsed values', function() { + // A value that ends in a truncated CP932 multi-byte sequence... + const truncated = Buffer.concat([iconv.encode('テスト', 'CP932'), Buffer.from([0x82])]); + const first = readValue(buildCharBuffer(truncated), 0, buildCharMetadata(dataTypeByName.VarChar, 'CP932'), utcOptions); + assert.strictEqual(first.value, iconv.decode(truncated, 'CP932')); + + // ...must not leak into the decoding of the next value. + const second = readValue(buildCharBuffer(Buffer.from([0x82, 0xA0, 0x41])), 0, buildCharMetadata(dataTypeByName.VarChar, 'CP932'), utcOptions); + assert.strictEqual(second.value, 'あA'); + }); + + it('should parse values that are ASCII except for the final byte', function() { + const buf = buildCharBuffer(Buffer.from([0x61, 0x62, 0x63, 0xFC])); // "abcü" in CP1252 + const result = readValue(buf, 0, buildCharMetadata(dataTypeByName.VarChar, 'CP1252'), utcOptions); + + assert.strictEqual(result.value, 'abcü'); + }); + }); });