Bug fixes for dbwire

stamparm · stamparm · commit a61e35980aa0 · 2026-07-12T11:36:35.000+02:00
diff --git a/extra/dbwire/clickhouse.py b/extra/dbwire/clickhouse.py
@@ -25,24 +25,34 @@
 from extra.dbwire import OperationalError
 from extra.dbwire import ProgrammingError
 
+# TabSeparated backslash escapes -> the literal byte they denote
+_ESCAPE = {ord("t"): 9, ord("n"): 10, ord("r"): 13, ord("0"): 0, ord("b"): 8,
+           ord("f"): 12, ord("a"): 7, ord("v"): 11, ord("\\"): 92, ord("'"): 39}
+
 def _unescape(value):
-    if value == "\\N":
+    # value: the raw bytes of one TSV field -> None (\N) or the unescaped bytes. Operates on bytes because a
+    # String/FixedString column can hold arbitrary non-UTF-8 data, which a whole-body utf-8 decode would destroy.
+    if value == b"\\N":
         return None
-    if "\\" not in value:
+    if b"\\" not in value:
         return value
-    out = []
-    i = 0
-    n = len(value)
+    src, out, i, n = bytearray(value), bytearray(), 0, len(value)
     while i < n:
-        ch = value[i]
-        if ch == "\\" and i + 1 < n:
-            nxt = value[i + 1]
-            out.append({"t": "\t", "n": "\n", "r": "\r", "0": "\0", "b": "\b", "f": "\f", "a": "\a", "v": "\v", "\\": "\\", "'": "'"}.get(nxt, nxt))
-            i += 2
+        c = src[i]
+        if c == 0x5c and i + 1 < n:  # backslash
+            out.append(_ESCAPE.get(src[i + 1], src[i + 1])); i += 2
         else:
-            out.append(ch)
-            i += 1
-    return "".join(out)
+            out.append(c); i += 1
+    return bytes(out)
+
+def _decode_cell(value):
+    # keep text as str; hand back raw bytes only when a value is not valid UTF-8 (sqlmap then hex-encodes it)
+    if value is None:
+        return None
+    try:
+        return value.decode("utf-8")
+    except UnicodeDecodeError:
+        return value
 
 class Cursor(object):
     def __init__(self, connection):
@@ -99,7 +109,7 @@ def close(self):
     def _query(self, query):
         req = Request(self._url, data=query.encode("utf-8"), headers=self._headers)
         try:
-            body = urlopen(req, timeout=self._timeout).read().decode("utf-8", "replace")
+            body = urlopen(req, timeout=self._timeout).read()  # bytes: column data may be non-UTF-8
         except HTTPError as ex:
             raise ProgrammingError("(remote) %s" % ex.read().decode("utf-8", "replace").strip())
         except URLError as ex:
@@ -109,13 +119,13 @@ def _query(self, query):
 
         if not body:
             return None, []
-        lines = body.split("\n")
-        if lines and lines[-1] == "":
+        lines = body.split(b"\n")
+        if lines and lines[-1] == b"":
             lines.pop()
         if not lines:
             return None, []
-        description = [(name, None, None, None, None, None, None) for name in (_unescape(_) for _ in lines[0].split("\t"))]
-        rows = [tuple(_unescape(_) for _ in line.split("\t")) for line in lines[1:]]
+        description = [(name, None, None, None, None, None, None) for name in (_decode_cell(_unescape(_)) for _ in lines[0].split(b"\t"))]
+        rows = [tuple(_decode_cell(_unescape(_)) for _ in line.split(b"\t")) for line in lines[1:]]
         return description, rows
 
 def connect(host=None, port=8123, user=None, password=None, database=None, connect_timeout=None, **kwargs):
diff --git a/extra/dbwire/mysql.py b/extra/dbwire/mysql.py
@@ -35,7 +35,11 @@
 
 _MAX_PACKET = 0x1000000
 _MAX_MESSAGE_LENGTH = 0x40000000  # cap on a (re-assembled) payload, to bound a hostile/corrupt stream
-_BINARY_CHARSET = 63              # collation id 63 == 'binary' (BLOB/BINARY/VARBINARY columns)
+_BINARY_CHARSET = 63              # collation id 63 == 'binary'
+# field types for which charset==63 genuinely denotes raw bytes (BLOB/BINARY/VARBINARY/BIT/GEOMETRY family).
+# Numeric & temporal columns ALSO report charset 63 in the text protocol, but carry their ASCII text form -
+# they must be decoded, not returned as bytes (else -d hexifies e.g. the int 12345 to '3132333435').
+_BINARY_TYPES = frozenset((15, 16, 249, 250, 251, 252, 253, 254, 255))  # VARCHAR,BIT,*BLOB,VAR_STRING,STRING,GEOMETRY
 
 def _xor(a, b):
     if str is bytes:  # Python 2
@@ -210,8 +214,9 @@ def _read_query_response(self):
             _, off = _lenc_str(cpay, off)             # org_name
             _, off = _lenc_int(cpay, off)             # length of the fixed-length block (0x0c)
             charset = struct.unpack("<H", cpay[off:off + 2])[0]
-            description.append((name.decode("utf-8", "replace"), None, None, None, None, None, None))
-            binary.append(charset == _BINARY_CHARSET)
+            col_type = _u8(cpay, off + 6)             # fixed block: charset(2) column_length(4) type(1) flags(2) ...
+            description.append((name.decode("utf-8", "replace"), col_type, None, None, None, None, None))
+            binary.append(charset == _BINARY_CHARSET and col_type in _BINARY_TYPES)
 
         _read_packet(self._sock)  # EOF after the column definitions
 
diff --git a/extra/dbwire/tds.py b/extra/dbwire/tds.py
@@ -151,35 +151,192 @@ def _decode_datetime(raw):
     import datetime
     return "%s" % (datetime.datetime(1900, 1, 1) + datetime.timedelta(days=days, milliseconds=ticks * 10.0 / 3.0))
 
+def _decode_money(raw):
+    if len(raw) == 4:
+        v = struct.unpack("<i", raw)[0]
+    else:  # 8 bytes: signed high dword, unsigned low dword
+        hi, lo = struct.unpack("<iI", raw)
+        v = (hi << 32) | lo
+    return "%.4f" % (v / 10000.0)
+
+def _decode_numeric(raw, scale):
+    sign = bytearray(raw)[0]  # 1 == positive, 0 == negative; magnitude is little-endian
+    magnitude = 0
+    for b in reversed(bytearray(raw[1:])):
+        magnitude = (magnitude << 8) | b
+    value = magnitude if sign else -magnitude
+    if scale:
+        s = "%0*d" % (scale + 1, abs(value))
+        return ("-" if value < 0 else "") + s[:-scale] + "." + s[-scale:]
+    return str(value)
+
+def _decode_guid(raw):
+    a, b, c = struct.unpack("<IHH", raw[:8])  # Data1/2/3 little-endian, Data4 big-endian (mixed-endian GUID)
+    d = raw[8:]
+    return "%08X-%04X-%04X-%s-%s" % (a, b, c,
+        "".join("%02X" % x for x in bytearray(d[:2])), "".join("%02X" % x for x in bytearray(d[2:])))
+
+def _decode_temporal(t, scale, raw):
+    # DATE 0x28 / TIME 0x29 / DATETIME2 0x2a / DATETIMEOFFSET 0x2b; time is scaled 10^-scale second units,
+    # rendered with the column's exact fractional precision (Python datetime only holds microseconds)
+    import datetime
+    offset = None
+    if t == 0x2b:  # trailing 2-byte signed offset (minutes from UTC); value bytes are stored as UTC
+        offset = struct.unpack("<h", raw[-2:])[0]
+        raw = raw[:-2]
+    has_date = t in (0x28, 0x2a, 0x2b)
+    has_time = t != 0x28
+    date_bytes = raw[-3:] if has_date else b""
+    time_bytes = (raw[:-3] if has_date else raw) if has_time else b""
+    days = struct.unpack("<I", date_bytes + b"\x00")[0] if has_date else 0
+    base = datetime.datetime(1, 1, 1) + datetime.timedelta(days=days)
+    frac = ""
+    if has_time:
+        ticks = struct.unpack("<Q", time_bytes + b"\x00" * (8 - len(time_bytes)))[0]
+        base += datetime.timedelta(seconds=ticks // (10 ** scale))
+        if scale:
+            frac = "." + ("%0*d" % (scale, ticks % (10 ** scale)))
+    if offset is not None:
+        base += datetime.timedelta(minutes=offset)
+    if t == 0x28:
+        return "%s" % base.date()
+    if t == 0x29:
+        return "%s" % base.time() + frac
+    s = "%s" % base + frac
+    if offset is not None:
+        sign, mins = ("+", offset) if offset >= 0 else ("-", -offset)
+        s += " %s%02d:%02d" % (sign, mins // 60, mins % 60)
+    return s
+
+# SQL Server COLLATION -> Python codec. The 5-byte collation is a little-endian uint32 (low 20 bits = LCID)
+# plus a 1-byte sort id: a non-zero sort id fixes the code page, else the LCID does. Only single-byte / DBCS
+# code pages need a codec (NVARCHAR is UTF-16, handled separately). Derived from pytds; default cp1252 (the
+# stock SQL_Latin1_General code page - NOT latin-1, whose 0x80-0x9F differ, corrupting e.g. the euro sign).
+_LCID_CP = {
+    0x405: "cp1250", 0x40e: "cp1250", 0x415: "cp1250", 0x418: "cp1250", 0x41a: "cp1250", 0x41b: "cp1250",
+    0x41c: "cp1250", 0x424: "cp1250", 0x402: "cp1251", 0x419: "cp1251", 0x422: "cp1251", 0x423: "cp1251",
+    0x42f: "cp1251", 0x408: "cp1253", 0x41f: "cp1254", 0x42c: "cp1254", 0x443: "cp1254", 0x40d: "cp1255",
+    0x401: "cp1256", 0x420: "cp1256", 0x429: "cp1256", 0x425: "cp1257", 0x426: "cp1257", 0x427: "cp1257",
+    0x42a: "cp1258", 0x41e: "cp874", 0x411: "cp932", 0x804: "cp936", 0x1004: "cp936", 0x412: "cp949",
+    0x404: "cp950", 0xc04: "cp950", 0x1404: "cp950",
+}
+
+def _sortid_cp(sid):
+    if 30 <= sid <= 34:
+        return "cp437"
+    if 40 <= sid <= 44 or sid == 49 or 55 <= sid <= 61:
+        return "cp850"
+    if sid in (51, 52, 53, 54) or 183 <= sid <= 186:
+        return "cp1252"
+    if 80 <= sid <= 96:
+        return "cp1250"
+    if 104 <= sid <= 108:
+        return "cp1251"
+    if 112 <= sid <= 124:
+        return "cp1253"
+    if 128 <= sid <= 130:
+        return "cp1254"
+    if 136 <= sid <= 138:
+        return "cp1255"
+    if 144 <= sid <= 146:
+        return "cp1256"
+    if 152 <= sid <= 160:
+        return "cp1257"
+    return None
+
+def _collation_codec(collation):
+    if not collation or len(collation) < 5:
+        return "cp1252"
+    lump = struct.unpack("<I", collation[:4])[0]
+    sid = bytearray(collation)[4]
+    if sid:
+        return _sortid_cp(sid) or "cp1252"
+    return _LCID_CP.get(lump & 0xfffff, "cp1252")
+
+def _decode_variant(body):
+    # SQL_VARIANT value: base type (1) | property-bytes count (1) | type-specific metadata | value bytes
+    b = bytearray(body)
+    base, propbytes = b[0], b[1]
+    meta, val = body[2:2 + propbytes], body[2 + propbytes:]
+    if base == 0x30:
+        return str(bytearray(val)[0])
+    if base == 0x32:
+        return "1" if bytearray(val)[0] else "0"
+    if base == 0x34:
+        return str(struct.unpack("<h", val)[0])
+    if base == 0x38:
+        return str(struct.unpack("<i", val)[0])
+    if base == 0x7f:
+        return str(struct.unpack("<q", val)[0])
+    if base == 0x3b:
+        return repr(struct.unpack("<f", val)[0])
+    if base == 0x3e:
+        return repr(struct.unpack("<d", val)[0])
+    if base in (0x3c, 0x7a):
+        return _decode_money(val)
+    if base == 0x3d:
+        return _decode_datetime(val)
+    if base == 0x24:
+        return _decode_guid(val)
+    if base in (0x6a, 0x6c):  # decimal/numeric: metadata = precision, scale
+        return _decode_numeric(val, bytearray(meta)[1])
+    if base in (0xe7, 0xef):
+        return val.decode("utf-16-le", "replace")
+    if base in (0xa5, 0xad):
+        return val  # binary -> raw bytes
+    if base in (0xa7, 0xaf):  # (var)char: metadata = 5-byte collation + 2-byte max length
+        return val.decode(_collation_codec(meta[:5]), "replace")
+    if base == 0x28:
+        return _decode_temporal(base, 0, val)
+    if base in (0x29, 0x2a, 0x2b):  # metadata = scale
+        return _decode_temporal(base, bytearray(meta)[0], val)
+    return "".join("%02x" % x for x in bytearray(val))  # unknown base type -> hex (never desyncs)
+
 class _Column(object):
-    __slots__ = ("name", "type", "size", "scale", "binary")
+    __slots__ = ("name", "type", "size", "scale", "binary", "collation")
 
 def _parse_type_info(data, off):
     col = _Column()
     col.type = _u8(data, off); off += 1
-    col.size, col.scale, col.binary = 0, 0, False
+    col.size, col.scale, col.binary, col.collation = 0, 0, False, None
     t = col.type
     if t in (0x30, 0x32, 0x34, 0x38, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x7a, 0x7f, 0x1f):
         pass  # fixed-length types, size implied by type
-    elif t in (0x26, 0x68, 0x6d, 0x6e, 0x6f, 0x24, 0x2e, 0x37):  # INTN/BITN/FLTN/MONEYN/DATETIMN/GUID
+    elif t in (0x26, 0x68, 0x6d, 0x6e, 0x6f, 0x24):  # INTN/BITN/FLTN/MONEYN/DATETIMN/GUID
         col.size = _u8(data, off); off += 1
-    elif t in (0x6a, 0x6c):  # DECIMALN / NUMERICN
+    elif t in (0x6a, 0x6c, 0x37, 0x3f):  # DECIMALN/NUMERICN + legacy DECIMAL/NUMERIC (size, precision, scale)
         col.size = _u8(data, off); off += 1
         off += 1  # precision
         col.scale = _u8(data, off); off += 1
     elif t in (0xa7, 0xaf, 0xe7, 0xef):  # (BIG)VARCHAR/CHAR, N(VAR)CHAR
         col.size = struct.unpack("<H", data[off:off + 2])[0]; off += 2
-        off += 5  # collation
+        col.collation = data[off:off + 5]; off += 5
     elif t in (0xa5, 0xad):  # (BIG)VARBINARY / BINARY
         col.size = struct.unpack("<H", data[off:off + 2])[0]; off += 2
         col.binary = True
     elif t in (0x28, 0x29, 0x2a, 0x2b):  # DATE/TIME/DATETIME2/DATETIMEOFFSET
         if t != 0x28:
             col.scale = _u8(data, off); off += 1
+    elif t == 0xf0:  # UDT (CLR geometry/geography/hierarchyid) - value arrives as PLP raw bytes
+        off += 2  # max byte size
+        for _ in range(3):  # db, schema, type name: B_VARCHAR
+            off += 1 + _u8(data, off) * 2
+        off += 2 + struct.unpack("<H", data[off:off + 2])[0] * 2  # assembly-qualified name: US_VARCHAR
+        col.binary = True
+    elif t == 0xf1:  # XML (value arrives PLP-encoded UTF-16, no size in TYPE_INFO)
+        if _u8(data, off):  # schema-present: B_VARCHAR dbname, B_VARCHAR owner, US_VARCHAR collection
+            off += 1
+            for _ in range(2):
+                off += 1 + _u8(data, off) * 2
+            off += 2 + struct.unpack("<H", data[off:off + 2])[0] * 2
+        else:
+            off += 1
+    elif t == 0x62:  # SQL_VARIANT (4-byte max length; per-value base type carried in the body)
+        col.size = struct.unpack("<i", data[off:off + 4])[0]; off += 4
     elif t in (0x23, 0x63, 0x22):  # TEXT/NTEXT/IMAGE
         col.size = struct.unpack("<i", data[off:off + 4])[0]; off += 4
         if t in (0x23, 0x63):
-            off += 5  # collation
+            col.collation = data[off:off + 5]; off += 5
         col.binary = (t == 0x22)
         # table name (num parts + parts) follows in COLMETADATA for these; handled by caller via name read
     else:
@@ -219,10 +376,9 @@ def _decode_value(col, data, off):
     if t == 0x3e:
         return repr(struct.unpack("<d", data[off:off + 8])[0]), off + 8
     if t == 0x7a:  # MONEY4 (4 bytes)
-        return "%.4f" % (struct.unpack("<i", data[off:off + 4])[0] / 10000.0), off + 4
-    if t == 0x3c:  # MONEY (8 bytes: high 4 then low 4)
-        hi, lo = struct.unpack("<iI", data[off:off + 8])
-        return "%.4f" % (((hi << 32) | lo) / 10000.0), off + 8
+        return _decode_money(data[off:off + 4]), off + 4
+    if t == 0x3c:  # MONEY (8 bytes: high dword then low dword)
+        return _decode_money(data[off:off + 8]), off + 8
     if t == 0x3d:  # DATETIME (8 bytes)
         return _decode_datetime(data[off:off + 8]), off + 8
     if t == 0x3a:  # DATETIM4 / smalldatetime (2-byte days since 1900 + 2-byte minutes)
@@ -245,7 +401,7 @@ def _decode_value(col, data, off):
             return raw, off  # binary -> raw bytes
         if t in (0xe7, 0xef):
             return raw.decode("utf-16-le", "replace"), off
-        return raw.decode("latin-1"), off  # (var)char is the collation's single-byte codepage; latin-1 round-trips every byte losslessly
+        return raw.decode(_collation_codec(col.collation), "replace"), off  # (var)char: the collation's code page
 
     if t in (0x23, 0x63, 0x22):  # TEXT/NTEXT/IMAGE: 1-byte textptr len (0 = NULL) then textptr+timestamp then 4-byte len
         ptr_len = _u8(data, off); off += 1
@@ -258,7 +414,21 @@ def _decode_value(col, data, off):
             return raw, off
         if t == 0x63:
             return raw.decode("utf-16-le", "replace"), off
-        return raw.decode("latin-1"), off  # TEXT is single-byte codepage; latin-1 is lossless
+        return raw.decode(_collation_codec(col.collation), "replace"), off  # TEXT: the collation's code page
+
+    if t == 0xf1:  # XML: PLP-encoded UTF-16-LE
+        raw, off = _read_plp(data, off)
+        return (raw.decode("utf-16-le", "replace") if raw is not None else None), off
+
+    if t == 0xf0:  # UDT (geometry/geography/hierarchyid): PLP raw bytes -> sqlmap hex-encodes them
+        return _read_plp(data, off)
+
+    if t == 0x62:  # SQL_VARIANT: 4-byte total length (0 = NULL) then a self-describing value body
+        (total,) = struct.unpack("<i", data[off:off + 4]); off += 4
+        if total <= 0:
+            return None, off
+        body, off = data[off:off + total], off + total
+        return _decode_variant(body), off
 
     # nullable / length-prefixed numeric & misc
     (n,) = struct.unpack("<B", data[off:off + 1]); off += 1
@@ -272,47 +442,20 @@ def _decode_value(col, data, off):
     if t == 0x6d:  # FLTN
         return repr(struct.unpack("<f" if n == 4 else "<d", raw)[0]), off
     if t in (0x6e, 0x3d, 0x7a):  # MONEYN / MONEY / MONEY4
-        if n == 4:
-            v = struct.unpack("<i", raw)[0]
-        else:
-            hi, lo = struct.unpack("<ii", raw)
-            v = (hi << 32) | (lo & 0xffffffff)
-        return "%.4f" % (v / 10000.0), off
-    if t in (0x6a, 0x6c):  # DECIMALN / NUMERICN
-        sign = bytearray(raw)[0]
-        magnitude = 0
-        for b in reversed(bytearray(raw[1:])):
-            magnitude = (magnitude << 8) | b
-        value = magnitude if sign else -magnitude
-        if col.scale:
-            s = "%0*d" % (col.scale + 1, value if value >= 0 else -value)
-            s = ("-" if value < 0 else "") + s[:-col.scale] + "." + s[-col.scale:]
-            return s, off
-        return str(value), off
+        return _decode_money(raw), off
+    if t in (0x6a, 0x6c, 0x37, 0x3f):  # DECIMALN / NUMERICN (+ legacy DECIMAL / NUMERIC)
+        return _decode_numeric(raw, col.scale), off
     if t == 0x24:  # GUID
-        a, b, c = struct.unpack("<IHH", raw[:8])
-        d = raw[8:]
-        return ("%08X-%04X-%04X-%s-%s" % (a, b, c, "".join("%02X" % x for x in bytearray(d[:2])), "".join("%02X" % x for x in bytearray(d[2:])))), off
+        return _decode_guid(raw), off
     if t == 0x6f:  # DATETIMN (n=8 datetime, n=4 smalldatetime)
         if n == 8:
             return _decode_datetime(raw), off
         import datetime
         days, mins = struct.unpack("<HH", raw)
         return "%s" % (datetime.datetime(1900, 1, 1) + datetime.timedelta(days=days, minutes=mins)), off
-    if t == 0x28:  # DATE (3 bytes: days since 0001-01-01)
-        import datetime
-        days = struct.unpack("<I", raw[:3] + b"\x00")[0]
-        return "%s" % (datetime.date(1, 1, 1) + datetime.timedelta(days=days)), off
-    if t in (0x2a, 0x29):  # DATETIME2 (time + 3-byte date) / TIME (time only), time = scaled 10^-scale s units
-        import datetime
-        date_bytes = raw[-3:] if t == 0x2a else b"\x00\x00\x00"
-        time_bytes = raw[:-3] if t == 0x2a else raw
-        days = struct.unpack("<I", date_bytes + b"\x00")[0]
-        ticks = struct.unpack("<Q", time_bytes + b"\x00" * (8 - len(time_bytes)))[0]
-        seconds = ticks / (10.0 ** (col.scale or 7))
-        base = datetime.datetime(1, 1, 1) + datetime.timedelta(days=days, seconds=seconds)
-        return ("%s" % base if t == 0x2a else "%s" % base.time()), off
-    # unknown date/time layout: return the raw hex as a last resort (never desyncs)
+    if t in (0x28, 0x29, 0x2a, 0x2b):  # DATE / TIME / DATETIME2 / DATETIMEOFFSET
+        return _decode_temporal(t, col.scale, raw), off
+    # unknown layout: return the raw hex as a last resort (never desyncs)
     return "".join("%02x" % x for x in bytearray(raw)), off
 
 def _parse_tokens(sock, login=False):
diff --git a/lib/core/settings.py b/lib/core/settings.py
