Added support for FIXED types (SAP#261)

- Added functions to read and write FIXED8, FIXED12, and FIXED16 types
- Modified Statement.js and Writer.js to pass the fraction metadata to the Writer for use in writing FIXED types
- Changed the behaviour when writing fixed DECIMAL types with inputs
larger than the max mantissa length to truncate rather than rounding
    - The behaviour for writing floating DECIMAL types larger than
      the max mantissa length remains to round (consistent with HANA's
      TO_DECIMAL)
- Added a zero to the zeropad.js to fix a case where large values inserted
from FIXED types could yield '1undefined' when returning as a DECIMAL
- Added a description for data format version 8 and its changes

Test Changes
- Added unit tests for the new FIXED type functions in bignum.js, Reader.js and Writer.js
- Added integration tests for the new FIXED types
    - The tests for FIXED types also run on DFV 7 to ensure the same behaviour for DECIMAL types

Author: he-is-harry

lib/protocol/ExecuteTask.js

@@ -30,7 +30,10 @@ function ExecuteTask(connection, options, callback) {
   this.scrollableCursor = options.scrollableCursor;
   this.statementId = options.statementId;
   this.functionCode = options.functionCode;
-  this.writer = new Writer(options.parameters.types, connection.useCesu8, connection.spatialTypes);
+  this.writer = new Writer(
+    { types: options.parameters.types, fractions: options.parameters.fractions },
+    { useCesu8: connection.useCesu8, spatialTypes: connection.spatialTypes }
+  );
   var values = options.parameters.values;
   if (values.length && Array.isArray(values[0])) {
     this.parameterValues = values.slice();

lib/protocol/Parser.js

@@ -109,7 +109,8 @@ function createHonestParseFunction(metadata, options) {
 
   function addReadFunction(column) {
     var args = [];
-    if (column.dataType === TypeCode.DECIMAL) {
+    if (column.dataType === TypeCode.DECIMAL || column.dataType === TypeCode.FIXED8
+      || column.dataType === TypeCode.FIXED12 || column.dataType === TypeCode.FIXED16) {
       args.push(column.fraction);
     }
     column.f = {
@@ -154,7 +155,8 @@ function createFunctionBody(metadata, options) {
 
   function getReadFunction(column) {
     var fn = ReadFunction[column.dataType];
-    if (column.dataType === TypeCode.DECIMAL) {
+    if (column.dataType === TypeCode.DECIMAL || column.dataType === TypeCode.FIXED8
+      || column.dataType === TypeCode.FIXED12 || column.dataType === TypeCode.FIXED16) {
       fn += '(' + column.fraction + ')';
     } else {
       fn += '()';

lib/protocol/Reader.js

@@ -365,6 +365,33 @@ Reader.prototype.readDecimal = function readDecimal(fraction) {
   return value;
 };
 
+Reader.prototype.readFixed8 = function readFixed8(fraction) {
+  if (this.buffer[this.offset++] === 0x00) {
+    return null;
+  }
+  var value = bignum.readFIXED(this.buffer, 8, this.offset, fraction);
+  this.offset += 8;
+  return value;
+}
+
+Reader.prototype.readFixed12 = function readFixed12(fraction) {
+  if (this.buffer[this.offset++] === 0x00) {
+    return null;
+  }
+  var value = bignum.readFIXED(this.buffer, 12, this.offset, fraction);
+  this.offset += 12;
+  return value;
+}
+
+Reader.prototype.readFixed16 = function readFixed16(fraction) {
+  if (this.buffer[this.offset++] === 0x00) {
+    return null;
+  }
+  var value = bignum.readFIXED(this.buffer, 16, this.offset, fraction);
+  this.offset += 16;
+  return value;
+}
+
 Reader.prototype.readAlphanum = function readAlphanum() {
   return this.readBytes(false, true);
 }

lib/protocol/Statement.js

@@ -112,9 +112,14 @@ Statement.prototype._normalizeInputParameters = function _normalizeInputParamete
     return values[metadata.name];
   }
 
+  function getTypeFraction(metadata) {
+    return metadata.fraction;
+  }
+
   var parameters = {
     types: inputParameterMetadata.map(getDataType),
-    values: undefined
+    values: undefined,
+    fractions: inputParameterMetadata.map(getTypeFraction),
   };
 
   parameters.values = Array.isArray(values) ? values : inputParameterMetadata.filter(isDefined).map(getObjectValue);

lib/protocol/Writer.js

@@ -22,6 +22,7 @@ var LobOptions = common.LobOptions;
 var NormalizedTypeCode = common.NormalizedTypeCode;
 var bignum = util.bignum;
 var calendar = util.calendar;
+var zeropad = require('../util/zeropad');
 var isValidDay = calendar.isValidDay;
 var isValidTime = calendar.isValidTime;
 var isZeroDay = calendar.isZeroDay;
@@ -40,12 +41,22 @@ var REGEX = {
 };
 
 const maxDecimalMantissaLen = 34;
-
-function Writer(types, useCesu8, spatialTypes) {
-  this._types = types.map(normalizeType);
+const maxFixedMantissaLen = 38;
+
+/**
+ * Constructs a Writer to write input parameters into server readable representation
+ * @param {Object} params - Metadata for input parameters to be written to the server
+ * @param {number[]} params.types - Array of type codes for each parameter
+ * @param {number[]} params.fractions - Array of the fraction / scale of each parameter
+ * (fraction metadata is only necessary for FIXED / DECIMAL types)
+ * @param {Object} options - Stores options to modify the way data types are written
+ */
+function Writer(params, options) {
+  this._types = params.types.map(normalizeType);
+  this._fractions = params.fractions;
   this.reset();
-  this._useCesu8 = (useCesu8 === true);
-  this._spatialTypes = (spatialTypes === 1 ? 1 : 0);
+  this._useCesu8 = (options && options.useCesu8 === true);
+  this._spatialTypes = ((options && options.spatialTypes === 1) ? 1 : 0);
 }
 
 function normalizeType(type) {
@@ -67,13 +78,13 @@ Writer.prototype.reset = function reset() {
 Writer.prototype.setValues = function setValues(values) {
   this.reset();
   for (var i = 0; i < values.length; i++) {
-    this.add(this._types[i], values[i]);
+    this.add(this._types[i], values[i], this._fractions ? this._fractions[i] : undefined);
   }
   this._params = true;
 };
 
-exports.create = function createWriter(params, useCesu8, spatialTypes) {
-  var writer = new Writer(params.types, useCesu8, spatialTypes);
+exports.create = function createWriter(params, options) {
+  var writer = new Writer(params, options);
   writer.setValues(params.values);
   return writer;
 };
@@ -96,9 +107,12 @@ Object.defineProperties(Writer.prototype, {
   }
 });
 
-Writer.prototype.add = function add(type, value) {
+Writer.prototype.add = function add(type, value, fraction) {
   if (typeof value === 'undefined' || value === null) {
     this.pushNull(type);
+  } else if (type === TypeCode.DECIMAL || type === TypeCode.FIXED8
+    || type === TypeCode.FIXED12 || type === TypeCode.FIXED16) {
+    this[type](value, fraction);
   } else {
     this[type](value);
   }
@@ -533,12 +547,12 @@ Writer.prototype[TypeCode.DOUBLE] = function writeDouble(value) {
   this.push(buffer);
 };
 
-Writer.prototype[TypeCode.DECIMAL] = function writeDecimal(value) {
+Writer.prototype[TypeCode.DECIMAL] = function writeDecimal(value, fraction) {
   var decimal;
   if (util.isString(value)) {
-    decimal = stringToDecimal(value);
+    decimal = stringToDecimal(value, maxDecimalMantissaLen, fraction);
   } else if (util.isNumber(value)) {
-    decimal = stringToDecimal(value.toExponential());
+    decimal = stringToDecimal(value.toExponential(), maxDecimalMantissaLen, fraction);
   } else {
     throw createInputError('DECIMAL');
   }
@@ -548,6 +562,103 @@ Writer.prototype[TypeCode.DECIMAL] = function writeDecimal(value) {
   this.push(buffer);
 };
 
+function toFixedDecimal(value, fraction, typeStr) {
+  var decimal;
+  // Convert to decimal object with maximum number of digits 38
+  if (util.isString(value)) {
+    decimal = stringToDecimal(value, maxFixedMantissaLen, fraction, typeStr);
+  } else if (util.isNumber(value)) {
+    decimal = stringToDecimal(value.toExponential(), maxFixedMantissaLen, fraction, typeStr);
+  } else {
+    throw createInputError(typeStr);
+  }
+  // Truncate decimal with the minimum exponent being -fraction, so there are at most
+  // 'fraction' digits after the decimal
+  decimal = truncateDecimalToExp(decimal, -fraction);
+
+  if (decimal.m.length + decimal.e + fraction > maxFixedMantissaLen) {
+    throw createInputError(typeStr); // Numeric overflow, greater than maximum precision 
+  }
+
+  if ((-decimal.e) < fraction) {
+    decimal.m += zeropad.ZEROS[fraction + decimal.e];
+  }
+  return decimal;
+}
+
+function writeFixed16Buffer(decimal, buffer, offset) {
+  bignum.writeUInt128LE(buffer, decimal.m, offset);
+  if (decimal.s === -1) {
+    // Apply two's complement conversion
+    var extraOne = true;
+    for (var i = offset; i < offset + 16; i++) {
+      if (extraOne) {
+        if (buffer[i] !== 0) {
+          buffer[i] = 0xff - buffer[i] + 1;
+          extraOne = false;
+        } else {
+          buffer[i] = 0;
+        }
+      } else {
+        buffer[i] = 0xff - buffer[i];
+      }
+    }
+  }
+}
+
+function checkFixedOverflow(decimal, extBuffer, byteLimit, typeStr) {
+  if (decimal.s === -1) {
+    for (var i = byteLimit; i < 16; ++i) {
+      if (extBuffer[i] != 0xff) {
+        throw createInputError(typeStr);
+      }
+    }
+    if ((extBuffer[byteLimit - 1] & 0x80) == 0) {
+      throw createInputError(typeStr);
+    }
+  } else {
+    for (var i = byteLimit; i < 16; ++i) {
+      if (extBuffer[i] != 0) {
+        throw createInputError(typeStr);
+      }
+    }
+    if (extBuffer[byteLimit - 1] & 0x80) {
+      throw createInputError(typeStr);
+    }
+  }
+}
+
+Writer.prototype[TypeCode.FIXED8] = function writeFixed8(value, fraction) {
+  var extBuffer = new Buffer(16);
+  var decimal = toFixedDecimal(value, fraction, 'FIXED8');
+  writeFixed16Buffer(decimal, extBuffer, 0);
+  // Check that the representation does not exceed 8 bytes
+  checkFixedOverflow(decimal, extBuffer, 8, 'FIXED8');
+  var buffer = new Buffer(9);
+  buffer[0] = TypeCode.FIXED8;
+  extBuffer.copy(buffer, 1, 0, 8);
+  this.push(buffer);
+}
+
+Writer.prototype[TypeCode.FIXED12] = function writeFixed12(value, fraction) {
+  var extBuffer = new Buffer(16);
+  var decimal = toFixedDecimal(value, fraction, 'FIXED12');
+  writeFixed16Buffer(decimal, extBuffer, 0);
+  // Check that the representation does not exceed 12 bytes
+  checkFixedOverflow(decimal, extBuffer, 12, 'FIXED12');
+  var buffer = new Buffer(13);
+  buffer[0] = TypeCode.FIXED12;
+  extBuffer.copy(buffer, 1, 0, 12);
+  this.push(buffer);
+}
+
+Writer.prototype[TypeCode.FIXED16] = function writeFixed16(value, fraction) {
+  var buffer = new Buffer(17);
+  buffer[0] = TypeCode.FIXED16;
+  writeFixed16Buffer(toFixedDecimal(value, fraction, 'FIXED16'), buffer, 1);
+  this.push(buffer);
+}
+
 Writer.prototype[TypeCode.NSTRING] = function writeNString(value) {
   this.writeCharacters(TypeCode.NSTRING, value);
 };
@@ -870,12 +981,12 @@ function trimTrailingZeroes(str) {
   return str.substring(0, i + 1);
 }
 
-function stringToDecimal(str) {
+function stringToDecimal(str, maxMantissaLen, fraction, typeStr) {
   /* jshint bitwise:false */
   var dec = str.match(REGEX.DECIMAL);
   // REGEX.DECIMAL will match "." and "" despite these being invalid.
   if (!dec || str === "." || str === "") {
-    throw createInputError('DECIMAL');
+    throw createInputError(typeStr === undefined ? 'DECIMAL' : typeStr);
   }
   var sign = dec[1] === '-' ? -1 : 1;
   var mInt = dec[2] || '';
@@ -887,14 +998,17 @@ function stringToDecimal(str) {
   if(mantissa.length === 0) mantissa = "0";
   exp -= mFrac.length
 
-  // round to maxDecimalMantissaLen digits and increment exp appropriately
-  if(mantissa.length > maxDecimalMantissaLen) {
-    var followDigit = mantissa[maxDecimalMantissaLen];
-    exp += (mantissa.length - maxDecimalMantissaLen)
-    mantissa = mantissa.substring(0, maxDecimalMantissaLen);
-    if(followDigit > '4') {
+  // Fit to maxMantissaLen digits and increment exp appropriately
+  if(mantissa.length > maxMantissaLen) {
+    var followDigit = mantissa[maxMantissaLen];
+    exp += (mantissa.length - maxMantissaLen)
+    mantissa = mantissa.substring(0, maxMantissaLen);
+    // When writing a floating point decimal (fraction > maxMantissaLen or
+    // fraction === undefined), we round to the max mantissa size, but with
+    // fixed we truncate to the max mantissa size
+    if((fraction === undefined || fraction > maxMantissaLen) && followDigit > '4') {
       // round up
-      var i = maxDecimalMantissaLen - 1;
+      var i = maxMantissaLen - 1;
       while(i >= 0 && mantissa[i] === '9') {
         i -= 1;
       }
@@ -907,9 +1021,9 @@ function stringToDecimal(str) {
         mantissa = mantissa.substring(0, i + 1);
         mantissa = setChar(mantissa, i, String.fromCharCode(mantissa.charCodeAt(i) + 1));
       }
-    } else if(mantissa[maxDecimalMantissaLen - 1] === '0') {
+    } else if(mantissa[maxMantissaLen - 1] === '0') {
       var trimmed = trimTrailingZeroes(mantissa);
-      exp += (maxDecimalMantissaLen - trimmed.length);
+      exp += (maxMantissaLen - trimmed.length);
       mantissa = trimmed;
     }
   }
@@ -921,6 +1035,34 @@ function stringToDecimal(str) {
   };
 }
 
+function truncateDecimalToExp(decimal, minExp) {
+  var mantissa = decimal.m;
+  var exp = decimal.e;
+  var calcMaxMantissaLength;
+  if (exp < minExp) {
+    // Shift the max mantissa length such that the exponent is minExp
+    calcMaxMantissaLength = exp + mantissa.length - minExp;
+    if (calcMaxMantissaLength <= 0) {
+      // All digits are truncated away
+      return {s: 1, m: "0", e: minExp};
+    }
+  }
+
+  // truncate to calcMaxMantissaLen digits and increment exp appropriately
+  if(calcMaxMantissaLength && mantissa.length > calcMaxMantissaLength) {
+    exp += (mantissa.length - calcMaxMantissaLength);
+    mantissa = mantissa.substring(0, calcMaxMantissaLength);
+    // No need to trim trailing zeros since FIXED types will add zeros to
+    // match minExp
+  }
+
+  return {
+    s: decimal.s,
+    m: mantissa,
+    e: exp
+  };
+}
+
 function createInputError(type) {
   return new Error(util.format('Wrong input for %s type', type));
 }

lib/protocol/common/DataFormatVersion.js

@@ -22,6 +22,7 @@ module.exports = {
   LEVEL5: 5,
   LEVEL6: 6,
   LEVEL7: 7,
+  LEVEL8: 8,
   // Maximum data format version supported by this driver
-  MAX_VERSION: 7,
+  MAX_VERSION: 8,
 };

lib/protocol/common/NormalizedTypeCode.js

@@ -30,6 +30,10 @@ NormalizedTypeCode[TypeCode.DOUBLE] = TypeCode.DOUBLE;
 NormalizedTypeCode[TypeCode.REAL] = TypeCode.REAL;
 // Decimal
 NormalizedTypeCode[TypeCode.DECIMAL] = TypeCode.DECIMAL;
+// Fixed
+NormalizedTypeCode[TypeCode.FIXED8] = TypeCode.FIXED8;
+NormalizedTypeCode[TypeCode.FIXED12] = TypeCode.FIXED12;
+NormalizedTypeCode[TypeCode.FIXED16] = TypeCode.FIXED16;
 // String
 NormalizedTypeCode[TypeCode.STRING] = TypeCode.STRING;
 NormalizedTypeCode[TypeCode.VARCHAR1] = TypeCode.STRING;

lib/protocol/common/ReadFunction.js

@@ -38,6 +38,9 @@ var READ_FLOAT = 'readFloat';
 var READ_DECIMAL = 'readDecimal';
 var READ_ALPHANUM = 'readAlphanum';
 var READ_BOOLEAN = 'readBoolean';
+var READ_FIXED8 = 'readFixed8';
+var READ_FIXED12 = 'readFixed12';
+var READ_FIXED16 = 'readFixed16';
 
 ReadFunction[TypeCode.TINYINT] = READ_TINYINT;
 ReadFunction[TypeCode.SMALLINT] = READ_SMALLINT;
@@ -75,3 +78,6 @@ ReadFunction[TypeCode.DECIMAL] = READ_DECIMAL;
 ReadFunction[TypeCode.ST_GEOMETRY] = READ_BINARY;
 ReadFunction[TypeCode.ST_POINT] = READ_BINARY;
 ReadFunction[TypeCode.BOOLEAN] = READ_BOOLEAN;
+ReadFunction[TypeCode.FIXED8] = READ_FIXED8;
+ReadFunction[TypeCode.FIXED12] = READ_FIXED12;
+ReadFunction[TypeCode.FIXED16] = READ_FIXED16;