From 876abe87300a0e19f29cd2b247ca0ea2903392be Mon Sep 17 00:00:00 2001 From: "Brian M. Carlson" Date: Sat, 15 Mar 2014 15:36:27 -0500 Subject: [PATCH 1/2] Break type parsing into separate module --- lib/defaults.js | 2 +- lib/index.js | 2 - lib/native/query.js | 1 - lib/query.js | 1 - lib/result.js | 2 +- lib/types/arrayParser.js | 97 ----- lib/types/binaryParsers.js | 256 ------------- lib/types/index.js | 44 --- lib/types/textParsers.js | 238 ------------ package.json | 3 +- test/integration/client/huge-numeric-tests.js | 2 +- test/unit/client/query-tests.js | 74 ---- test/unit/client/typed-query-results-tests.js | 361 ------------------ test/unit/utils-tests.js | 25 +- 14 files changed, 15 insertions(+), 1093 deletions(-) delete mode 100644 lib/types/arrayParser.js delete mode 100644 lib/types/binaryParsers.js delete mode 100644 lib/types/index.js delete mode 100644 lib/types/textParsers.js delete mode 100644 test/unit/client/query-tests.js delete mode 100644 test/unit/client/typed-query-results-tests.js diff --git a/lib/defaults.js b/lib/defaults.js index 1241703fd..9f5687b05 100644 --- a/lib/defaults.js +++ b/lib/defaults.js @@ -46,5 +46,5 @@ var defaults = module.exports = { //parse int8 so you can get your count values as actual numbers module.exports.__defineSetter__("parseInt8", function(val) { - require('./types').setTypeParser(20, 'text', val ? parseInt : function(val) { return val; }); + require('pg-types').setTypeParser(20, 'text', val ? parseInt : function(val) { return val; }); }); diff --git a/lib/index.js b/lib/index.js index 423c3ed30..0cad644d7 100644 --- a/lib/index.js +++ b/lib/index.js @@ -3,7 +3,6 @@ var util = require('util'); var Client = require(__dirname+'/client'); var defaults = require(__dirname + '/defaults'); var pool = require(__dirname + '/pool'); -var types = require(__dirname + '/types/'); var Connection = require(__dirname + '/connection'); var PG = function(clientConstructor) { @@ -12,7 +11,6 @@ var PG = function(clientConstructor) { this.Client = pool.Client = clientConstructor; this.Query = this.Client.Query; this.pools = pool; - this.types = types; this.Connection = Connection; }; diff --git a/lib/native/query.js b/lib/native/query.js index d2b06b8e9..9b1a730bc 100644 --- a/lib/native/query.js +++ b/lib/native/query.js @@ -1,7 +1,6 @@ var EventEmitter = require('events').EventEmitter; var util = require('util'); -var types = require(__dirname + '/../types/'); var utils = require(__dirname + '/../utils'); var Result = require(__dirname + '/../result'); diff --git a/lib/query.js b/lib/query.js index c8d2db1eb..c595eda6b 100644 --- a/lib/query.js +++ b/lib/query.js @@ -2,7 +2,6 @@ var EventEmitter = require('events').EventEmitter; var util = require('util'); var Result = require(__dirname + '/result'); -var Types = require(__dirname + '/types/'); var utils = require(__dirname + '/utils'); var Query = function(config, values, callback) { diff --git a/lib/result.js b/lib/result.js index 8ec3de01f..f6c33cc42 100644 --- a/lib/result.js +++ b/lib/result.js @@ -1,4 +1,4 @@ -var types = require(__dirname + '/types/'); +var types = require('pg-types'); //result object returned from query //in the 'end' event and also diff --git a/lib/types/arrayParser.js b/lib/types/arrayParser.js deleted file mode 100644 index 96a37b93b..000000000 --- a/lib/types/arrayParser.js +++ /dev/null @@ -1,97 +0,0 @@ -function ArrayParser(source, converter) { - this.source = source; - this.converter = converter; - this.pos = 0; - this.entries = []; - this.recorded = []; - this.dimension = 0; - if (!this.converter) { - this.converter = function(entry) { - return entry; - }; - } -} - -ArrayParser.prototype.eof = function() { - return this.pos >= this.source.length; -}; - -ArrayParser.prototype.nextChar = function() { - var c; - if ((c = this.source[this.pos++]) === "\\") { - return { - char: this.source[this.pos++], - escaped: true - }; - } else { - return { - char: c, - escaped: false - }; - } -}; - -ArrayParser.prototype.record = function(c) { - return this.recorded.push(c); -}; - -ArrayParser.prototype.newEntry = function(includeEmpty) { - var entry; - if (this.recorded.length > 0 || includeEmpty) { - entry = this.recorded.join(""); - if (entry === "NULL" && !includeEmpty) { - entry = null; - } - if (entry !== null) { - entry = this.converter(entry); - } - this.entries.push(entry); - this.recorded = []; - } -}; - -ArrayParser.prototype.parse = function(nested) { - var c, p, quote; - if (nested === null) { - nested = false; - } - quote = false; - while (!this.eof()) { - c = this.nextChar(); - if (c.char === "{" && !quote) { - this.dimension++; - if (this.dimension > 1) { - p = new ArrayParser(this.source.substr(this.pos - 1), this.converter); - this.entries.push(p.parse(true)); - this.pos += p.pos - 2; - } - } else if (c.char === "}" && !quote) { - this.dimension--; - if (this.dimension === 0) { - this.newEntry(); - if (nested) { - return this.entries; - } - } - } else if (c.char === '"' && !c.escaped) { - if (quote) { - this.newEntry(true); - } - quote = !quote; - } else if (c.char === ',' && !quote) { - this.newEntry(); - } else { - this.record(c.char); - } - } - if (this.dimension !== 0) { - throw "array dimension not balanced"; - } - return this.entries; -}; - -module.exports = { - create: function(source, converter){ - return new ArrayParser(source, converter); - } -}; diff --git a/lib/types/binaryParsers.js b/lib/types/binaryParsers.js deleted file mode 100644 index a71ebb7cb..000000000 --- a/lib/types/binaryParsers.js +++ /dev/null @@ -1,256 +0,0 @@ -var parseBits = function(data, bits, offset, invert, callback) { - offset = offset || 0; - invert = invert || false; - callback = callback || function(lastValue, newValue, bits) { return (lastValue * Math.pow(2, bits)) + newValue; }; - var offsetBytes = offset >> 3; - - var inv = function(value) { - if (invert) { - return ~value & 0xff; - } - - return value; - }; - - // read first (maybe partial) byte - var mask = 0xff; - var firstBits = 8 - (offset % 8); - if (bits < firstBits) { - mask = (0xff << (8 - bits)) & 0xff; - firstBits = bits; - } - - if (offset) { - mask = mask >> (offset % 8); - } - - var result = 0; - if ((offset % 8) + bits >= 8) { - result = callback(0, inv(data[offsetBytes]) & mask, firstBits); - } - - // read bytes - var bytes = (bits + offset) >> 3; - for (var i = offsetBytes + 1; i < bytes; i++) { - result = callback(result, inv(data[i]), 8); - } - - // bits to read, that are not a complete byte - var lastBits = (bits + offset) % 8; - if (lastBits > 0) { - result = callback(result, inv(data[bytes]) >> (8 - lastBits), lastBits); - } - - return result; -}; - -var parseFloatFromBits = function(data, precisionBits, exponentBits) { - var bias = Math.pow(2, exponentBits - 1) - 1; - var sign = parseBits(data, 1); - var exponent = parseBits(data, exponentBits, 1); - - if (exponent === 0) { - return 0; - } - - // parse mantissa - var precisionBitsCounter = 1; - var parsePrecisionBits = function(lastValue, newValue, bits) { - if (lastValue === 0) { - lastValue = 1; - } - - for (var i = 1; i <= bits; i++) { - precisionBitsCounter /= 2; - if ((newValue & (0x1 << (bits - i))) > 0) { - lastValue += precisionBitsCounter; - } - } - - return lastValue; - }; - - var mantissa = parseBits(data, precisionBits, exponentBits + 1, false, parsePrecisionBits); - - // special cases - if (exponent == (Math.pow(2, exponentBits + 1) - 1)) { - if (mantissa === 0) { - return (sign === 0) ? Infinity : -Infinity; - } - - return NaN; - } - - // normale number - return ((sign === 0) ? 1 : -1) * Math.pow(2, exponent - bias) * mantissa; -}; - -var parseBool = function(value) { - return (parseBits(value, 8) == 1); -}; - -var parseInt16 = function(value) { - if (parseBits(value, 1) == 1) { - return -1 * (parseBits(value, 15, 1, true) + 1); - } - - return parseBits(value, 15, 1); -}; - -var parseInt32 = function(value) { - if (parseBits(value, 1) == 1) { - return -1 * (parseBits(value, 31, 1, true) + 1); - } - - return parseBits(value, 31, 1); -}; - -var parseFloat32 = function(value) { - return parseFloatFromBits(value, 23, 8); -}; - -var parseFloat64 = function(value) { - return parseFloatFromBits(value, 52, 11); -}; - -var parseNumeric = function(value) { - var sign = parseBits(value, 16, 32); - if (sign == 0xc000) { - return NaN; - } - - var weight = Math.pow(10000, parseBits(value, 16, 16)); - var result = 0; - - var digits = []; - var ndigits = parseBits(value, 16); - for (var i = 0; i < ndigits; i++) { - result += parseBits(value, 16, 64 + (16 * i)) * weight; - weight /= 10000; - } - - var scale = Math.pow(10, parseBits(value, 16, 48)); - return ((sign === 0) ? 1 : -1) * Math.round(result * scale) / scale; -}; - -var parseDate = function(isUTC, value) { - var sign = parseBits(value, 1); - var rawValue = parseBits(value, 63, 1); - - // discard usecs and shift from 2000 to 1970 - var result = new Date((((sign === 0) ? 1 : -1) * rawValue / 1000) + 946684800000); - - if (!isUTC) { - result.setTime(result.getTime() + result.getTimezoneOffset() * 60000); - } - - // add microseconds to the date - result.usec = rawValue % 1000; - result.getMicroSeconds = function() { - return this.usec; - }; - result.setMicroSeconds = function(value) { - this.usec = value; - }; - result.getUTCMicroSeconds = function() { - return this.usec; - }; - - return result; -}; - -var parseArray = function(value) { - var dim = parseBits(value, 32); - - var flags = parseBits(value, 32, 32); - var elementType = parseBits(value, 32, 64); - - var offset = 96; - var dims = []; - for (var i = 0; i < dim; i++) { - // parse dimension - dims[i] = parseBits(value, 32, offset); - offset += 32; - - // ignore lower bounds - offset += 32; - } - - var parseElement = function(elementType) { - // parse content length - var length = parseBits(value, 32, offset); - offset += 32; - - // parse null values - if (length == 0xffffffff) { - return null; - } - - var result; - if ((elementType == 0x17) || (elementType == 0x14)) { - // int/bigint - result = parseBits(value, length * 8, offset); - offset += length * 8; - return result; - } - else if (elementType == 0x19) { - // string - result = value.toString(this.encoding, offset >> 3, (offset += (length << 3)) >> 3); - return result; - } - else { - console.log("ERROR: ElementType not implemented: " + elementType); - } - }; - - var parse = function(dimension, elementType) { - var array = []; - var i; - - if (dimension.length > 1) { - var count = dimension.shift(); - for (i = 0; i < count; i++) { - array[i] = parse(dimension, elementType); - } - dimension.unshift(count); - } - else { - for (i = 0; i < dimension[0]; i++) { - array[i] = parseElement(elementType); - } - } - - return array; - }; - - return parse(dims, elementType); -}; - -var parseText = function(value) { - return value.toString('utf8'); -}; - -var parseBool = function(value) { - return (parseBits(value, 8) > 0); -}; - -var init = function(register) { - register(21, parseInt16); - register(23, parseInt32); - register(26, parseInt32); - register(1700, parseNumeric); - register(700, parseFloat32); - register(701, parseFloat64); - register(16, parseBool); - register(1114, parseDate.bind(null, false)); - register(1184, parseDate.bind(null, true)); - register(1007, parseArray); - register(1016, parseArray); - register(1008, parseArray); - register(1009, parseArray); - register(25, parseText); -}; - -module.exports = { - init: init -}; diff --git a/lib/types/index.js b/lib/types/index.js deleted file mode 100644 index 0750856c1..000000000 --- a/lib/types/index.js +++ /dev/null @@ -1,44 +0,0 @@ -var textParsers = require(__dirname + '/textParsers'); -var binaryParsers = require(__dirname + '/binaryParsers'); - -var typeParsers = { - text: {}, - binary: {} -}; - -//the empty parse function -var noParse = function(val) { - return String(val); -}; - -//returns a function used to convert a specific type (specified by -//oid) into a result javascript type -//note: the oid can be obtained via the following sql query: -//SELECT oid FROM pg_type WHERE typname = 'TYPE_NAME_HERE'; -var getTypeParser = function(oid, format) { - if (!typeParsers[format]) { - return noParse; - } - return typeParsers[format][oid] || noParse; -}; - -var setTypeParser = function(oid, format, parseFn) { - if(typeof format == 'function') { - parseFn = format; - format = 'text'; - } - typeParsers[format][oid] = parseFn; -}; - -textParsers.init(function(oid, converter) { - typeParsers.text[oid] = converter; -}); - -binaryParsers.init(function(oid, converter) { - typeParsers.binary[oid] = converter; -}); - -module.exports = { - getTypeParser: getTypeParser, - setTypeParser: setTypeParser -}; diff --git a/lib/types/textParsers.js b/lib/types/textParsers.js deleted file mode 100644 index cd16a1f98..000000000 --- a/lib/types/textParsers.js +++ /dev/null @@ -1,238 +0,0 @@ -var arrayParser = require(__dirname + "/arrayParser.js"); - -//parses PostgreSQL server formatted date strings into javascript date objects -var parseDate = function(isoDate) { - //TODO this could do w/ a refactor - var dateMatcher = /(\d{1,})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})(\.\d{1,})?/; - - var match = dateMatcher.exec(isoDate); - //could not parse date - if(!match) { - dateMatcher = /^(\d{1,})-(\d{2})-(\d{2})$/; - match = dateMatcher.test(isoDate); - if(!match) { - return null; - } else { - //it is a date in YYYY-MM-DD format - //add time portion to force js to parse as local time - return new Date(isoDate + ' 00:00:00'); - } - } - var isBC = /BC$/.test(isoDate); - var _year = parseInt(match[1], 10); - var isFirstCentury = (_year > 0) && (_year < 100); - var year = (isBC ? "-" : "") + match[1]; - - var month = parseInt(match[2],10)-1; - var day = match[3]; - var hour = parseInt(match[4],10); - var min = parseInt(match[5],10); - var seconds = parseInt(match[6], 10); - - var miliString = match[7]; - var mili = 0; - if(miliString) { - mili = 1000 * parseFloat(miliString); - } - - //match timezones like the following: - //Z (UTC) - //-05 - //+06:30 - var tZone = /([Z|+\-])(\d{2})?:?(\d{2})?/.exec(isoDate.split(' ')[1]); - //minutes to adjust for timezone - var tzAdjust = 0; - var date; - if(tZone) { - var type = tZone[1]; - switch(type) { - case 'Z': - break; - case '-': - tzAdjust = -(((parseInt(tZone[2],10)*60)+(parseInt(tZone[3]||0,10)))); - break; - case '+': - tzAdjust = (((parseInt(tZone[2],10)*60)+(parseInt(tZone[3]||0,10)))); - break; - default: - throw new Error("Unidentifed tZone part " + type); - } - - var utcOffset = Date.UTC(year, month, day, hour, min, seconds, mili); - - date = new Date(utcOffset - (tzAdjust * 60* 1000)); - } - //no timezone information - else { - date = new Date(year, month, day, hour, min, seconds, mili); - } - - if (isFirstCentury) { - date.setUTCFullYear(year); - } - - return date; -}; - -var parseBool = function(val) { - return val === 't'; -}; - -var parseIntegerArray = function(val) { - if(!val) { return null; } - var p = arrayParser.create(val, function(entry){ - if(entry !== null) { - entry = parseInt(entry, 10); - } - return entry; - }); - - return p.parse(); -}; - -var parseBigIntegerArray = function(val) { - if(!val) { return null; } - var p = arrayParser.create(val, function(entry){ - if(entry !== null) { - entry = parseBigInteger(entry).trim(); - } - return entry; - }); - - return p.parse(); -}; - -var parseFloatArray = function(val) { - if(!val) { return null; } - var p = arrayParser.create(val, function(entry) { - if(entry !== null) { - entry = parseFloat(entry); - } - return entry; - }); - - return p.parse(); -}; - -var parseStringArray = function(val) { - if(!val) { return null; } - - var p = arrayParser.create(val); - return p.parse(); -}; - - -var NUM = '([+-]?\\d+)'; -var YEAR = NUM + '\\s+years?'; -var MON = NUM + '\\s+mons?'; -var DAY = NUM + '\\s+days?'; -var TIME = '([+-])?(\\d\\d):(\\d\\d):(\\d\\d)'; -var INTERVAL = [YEAR,MON,DAY,TIME].map(function(p){ - return "("+p+")?"; -}).join('\\s*'); - -var parseInterval = function(val) { - if (!val) { return {}; } - var m = new RegExp(INTERVAL).exec(val); - var i = {}; - if (m[2]) { i.years = parseInt(m[2], 10); } - if (m[4]) { i.months = parseInt(m[4], 10); } - if (m[6]) { i.days = parseInt(m[6], 10); } - if (m[9]) { i.hours = parseInt(m[9], 10); } - if (m[10]) { i.minutes = parseInt(m[10], 10); } - if (m[11]) { i.seconds = parseInt(m[11], 10); } - if (m[8] == '-'){ - if (i.hours) { i.hours *= -1; } - if (i.minutes) { i.minutes *= -1; } - if (i.seconds) { i.seconds *= -1; } - } - for (var field in i){ - if (i[field] === 0) { - delete i[field]; - } - } - return i; -}; - -var parseByteA = function(val) { - if(/^\\x/.test(val)){ - // new 'hex' style response (pg >9.0) - return new Buffer(val.substr(2), 'hex'); - }else{ - var out = ""; - var i = 0; - while(i < val.length){ - if(val[i] != "\\"){ - out += val[i]; - ++i; - }else{ - if(val.substr(i+1,3).match(/[0-7]{3}/)){ - out += String.fromCharCode(parseInt(val.substr(i+1,3),8)); - i += 4; - }else{ - backslashes = 1; - while(i+backslashes < val.length && val[i+backslashes] == "\\") - backslashes++; - for(k=0; k Date: Sat, 15 Mar 2014 15:41:36 -0500 Subject: [PATCH 2/2] Add test for pg.types interface --- lib/index.js | 1 + test/unit/utils-tests.js | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/lib/index.js b/lib/index.js index 0cad644d7..3d7aa5807 100644 --- a/lib/index.js +++ b/lib/index.js @@ -12,6 +12,7 @@ var PG = function(clientConstructor) { this.Query = this.Client.Query; this.pools = pool; this.Connection = Connection; + this.types = require('pg-types'); }; util.inherits(PG, EventEmitter); diff --git a/test/unit/utils-tests.js b/test/unit/utils-tests.js index 660afbfbc..56c81dc52 100644 --- a/test/unit/utils-tests.js +++ b/test/unit/utils-tests.js @@ -2,6 +2,14 @@ require(__dirname + '/test-helper'); var utils = require(__dirname + "/../../lib/utils"); var defaults = require(__dirname + "/../../lib").defaults; + +test('ensure types is exported on root object', function() { + var pg = require('../../lib') + assert(pg.types) + assert(pg.types.getTypeParser) + assert(pg.types.setTypeParser) +}) + //this tests the monkey patching //to ensure comptability with older //versions of node