From 5f621d72c1f265bb7659b54eb33469db8a4443fd Mon Sep 17 00:00:00 2001 From: Nilabja Bhattacharya Date: Fri, 23 Sep 2022 03:33:17 +0530 Subject: [PATCH] fix(oracle): add support for Oracle DB 18c CI (#15016) * feat(oracle): add oracle dialect support (#1) * feat(oracle): add oracle dialect support * fix: addressing review comments (#7) * fix: addressing review comments * fix: minor fixes done (#9) * fix: minor fixes to the review comments * fix: merge from sequelize-v6 * fix: enable newly added unit tests for Oracle dialect * fix: remove dangling comma (#13) * fix: doc gen is fixed * fix: autogenerate the primary constraint name (#14) * fix: autogenerate the primary constraint name * fix: remove trailing comma * fix: make changes to ORADERBY as per v6 sync * fix: move test-unit-oracle above test-unit-all * fix: rename getInsertQueryReturnIntoBinds to populateInsertQueryReturnIntoBinds * fix: reorder parameters for function populateInsertQueryReturnIntoBinds * fix: incorporated review comments (#16) * fix: incorporated review comments * fix: modify string empty check with ! * feat: support for Oracle DB 18c * Oracle DB version change * added stop-oracle for 18 * fix: changes to DB version query * fix: cleanup * fix: describetable query fix * fix: dbVersion to remove round trip and 18.4 json test fix * fix: removed dbversion * fix: removed comment * fix: testing a feature * fix: testing a feature * fix: testing a feature * fix: testing a feature * fix: testing a feature * fix: testing a feature * fix: test * fix: test * fix: test * fix: test * fix: using semver to coerce version sring * fix: update to instant client latest version for oracle db 21c * fix: update to oracledb version in package.json * fix: update lockfile * fix: remove duplicate privileges.sql and wait-until-healthy.sh * fix: changes to start-oracle alias * fix: changes to start-oracle alias * fix: changes to start-oracle alias Co-authored-by: Sudarshan Soma <48428602+sudarshan12s@users.noreply.github.com> --- .github/workflows/ci.yml | 11 +++- dev/oracle/18-slim/docker-compose.yml | 17 ++++++ dev/oracle/18-slim/start.sh | 61 +++++++++++++++++++ dev/oracle/18-slim/stop.sh | 10 +++ dev/oracle/21-slim/start.sh | 12 ++-- dev/oracle/{21-slim => }/privileges.sql | 0 .../{21-slim => }/wait-until-healthy.sh | 0 package.json | 11 ++-- src/dialects/oracle/connection-manager.js | 3 + src/dialects/oracle/index.js | 2 +- src/dialects/oracle/query-generator.js | 4 +- src/dialects/oracle/query.js | 2 +- test/integration/json.test.js | 7 +++ yarn.lock | 8 +-- 14 files changed, 127 insertions(+), 21 deletions(-) create mode 100644 dev/oracle/18-slim/docker-compose.yml create mode 100755 dev/oracle/18-slim/start.sh create mode 100755 dev/oracle/18-slim/stop.sh rename dev/oracle/{21-slim => }/privileges.sql (100%) rename dev/oracle/{21-slim => }/wait-until-healthy.sh (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 26904850cc6a..2168fc9fa298 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,8 +51,9 @@ jobs: strategy: fail-fast: false matrix: + oracle-version: [18, 21] node-version: [10, 18] - name: Oracle DB (Node ${{ matrix.node-version }}) + name: Oracle DB ${{ matrix.oracle-version }} (Node ${{ matrix.node-version }}) runs-on: ubuntu-latest env: DIALECT: oracle @@ -69,8 +70,12 @@ jobs: with: node-version: ${{ matrix.node-version }} - run: yarn install --frozen-lockfile --ignore-engines - - name: Install Local Oracle DB - run: yarn start-oracle + - if: matrix.oracle-version == '18' + name: Install Local Oracle DB 18 + run: yarn start-oracle-oldest + - if: matrix.oracle-version == '21' + name: Install Local Oracle DB 21 + run: yarn start-oracle-latest - name: Unit Tests run: yarn test-unit - name: Integration Tests diff --git a/dev/oracle/18-slim/docker-compose.yml b/dev/oracle/18-slim/docker-compose.yml new file mode 100644 index 000000000000..1ba87e65e686 --- /dev/null +++ b/dev/oracle/18-slim/docker-compose.yml @@ -0,0 +1,17 @@ +# Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved + +services: + oraclexedb: + container_name: oraclexedb + image: gvenzl/oracle-xe:18-slim + environment: + ORACLE_PASSWORD: password + ports: + - 1521:1521 + healthcheck: + test: ["CMD-SHELL", "sqlplus", "system/password@XEPDB1"] + retries: 10 + +networks: + default: + name: sequelize-oraclexedb-network diff --git a/dev/oracle/18-slim/start.sh b/dev/oracle/18-slim/start.sh new file mode 100755 index 000000000000..fcd340bde51d --- /dev/null +++ b/dev/oracle/18-slim/start.sh @@ -0,0 +1,61 @@ +# Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved + +#!/usr/bin/env bash +set -Eeuxo pipefail # https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ +cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" # https://stackoverflow.com/a/17744637 + +# Remove an existing Oracle DB docker image +docker-compose -p oraclexedb down --remove-orphans + +# Bring up new Oracle DB docker image +docker-compose -p oraclexedb up -d + +# Wait until Oracle DB is set up and docker state is healthy +./../wait-until-healthy.sh oraclexedb + +# Moving privileges.sql to docker container +docker cp ../privileges.sql oraclexedb:/opt/oracle/. + +# Granting all the needed privileges to sequelizetest user +docker exec -t oraclexedb sqlplus system/password@XEPDB1 @privileges.sql + +SEQ_WORKSPACE="$PWD"/../../../ + +if [[ ! -d "$SEQ_WORKSPACE"/.oracle/ ]] +then + mkdir "$SEQ_WORKSPACE"/.oracle/ + if [[ $(uname) == 'Linux' ]] + then + wget https://download.oracle.com/otn_software/linux/instantclient/217000/instantclient-basic-linux.x64-21.7.0.0.0dbru.zip --no-check-certificate && + unzip instantclient-basic-linux.x64-21.7.0.0.0dbru.zip -d "$SEQ_WORKSPACE"/.oracle/ && + rm instantclient-basic-linux.x64-21.7.0.0.0dbru.zip && + mv "$SEQ_WORKSPACE"/.oracle/instantclient_21_7 "$SEQ_WORKSPACE"/.oracle/instantclient + + echo "Local Oracle instant client on Linux has been setup!" + elif [[ $(uname) == 'Darwin' ]] + then + if [[ ! -d ~/Downloads/instantclient_19_8 ]] + then + curl -O https://download.oracle.com/otn_software/mac/instantclient/198000/instantclient-basic-macos.x64-19.8.0.0.0dbru.dmg && + hdiutil mount instantclient-basic-macos.x64-19.8.0.0.0dbru.dmg && + /Volumes/instantclient-basic-macos.x64-19.8.0.0.0dbru/install_ic.sh && + hdiutil unmount /Volumes/instantclient-basic-macos.x64-19.8.0.0.0dbru && + rm instantclient-basic-macos.x64-19.8.0.0.0dbru.dmg && + mv ~/Downloads/instantclient_19_8/ "$SEQ_WORKSPACE"/.oracle/instantclient + else + cp -rf ~/Downloads/instantclient_19_8/ "$SEQ_WORKSPACE"/.oracle/instantclient + fi + ln -s "$SEQ_WORKSPACE"/.oracle/instantclient/libclntsh.dylib "$SEQ_WORKSPACE"/node_modules/oracledb/build/Release/ + + echo "Local Oracle instant client on macOS has been setup!" + else + # Windows + curl -O https://download.oracle.com/otn_software/nt/instantclient/216000/instantclient-basic-windows.x64-21.6.0.0.0dbru.zip && + unzip instantclient-basic-windows.x64-21.6.0.0.0dbru.zip -d "$SEQ_WORKSPACE"/.oracle/ && + rm instantclient-basic-windows.x64-21.6.0.0.0dbru.zip && + mv "$SEQ_WORKSPACE"/.oracle/instantclient_21_6/* "$SEQ_WORKSPACE"/node_modules/oracledb/build/Release + + echo "Local Oracle instant client on $(uname) has been setup!" + fi +fi +echo "Local Oracle DB is ready for use!" diff --git a/dev/oracle/18-slim/stop.sh b/dev/oracle/18-slim/stop.sh new file mode 100755 index 000000000000..694516a698ac --- /dev/null +++ b/dev/oracle/18-slim/stop.sh @@ -0,0 +1,10 @@ +# Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved + +#!/usr/bin/env bash +set -Eeuxo pipefail # https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ +cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" # https://stackoverflow.com/a/17744637 + + +docker-compose -p oraclexedb down --remove-orphans + +echo "Local Oracle DB instance stopped (if it was running)." diff --git a/dev/oracle/21-slim/start.sh b/dev/oracle/21-slim/start.sh index 5988e3be7acc..fcd340bde51d 100755 --- a/dev/oracle/21-slim/start.sh +++ b/dev/oracle/21-slim/start.sh @@ -11,10 +11,10 @@ docker-compose -p oraclexedb down --remove-orphans docker-compose -p oraclexedb up -d # Wait until Oracle DB is set up and docker state is healthy -./wait-until-healthy.sh oraclexedb +./../wait-until-healthy.sh oraclexedb # Moving privileges.sql to docker container -docker cp privileges.sql oraclexedb:/opt/oracle/. +docker cp ../privileges.sql oraclexedb:/opt/oracle/. # Granting all the needed privileges to sequelizetest user docker exec -t oraclexedb sqlplus system/password@XEPDB1 @privileges.sql @@ -26,10 +26,10 @@ then mkdir "$SEQ_WORKSPACE"/.oracle/ if [[ $(uname) == 'Linux' ]] then - wget https://download.oracle.com/otn_software/linux/instantclient/216000/instantclient-basic-linux.x64-21.6.0.0.0dbru.zip --no-check-certificate && - unzip instantclient-basic-linux.x64-21.6.0.0.0dbru.zip -d "$SEQ_WORKSPACE"/.oracle/ && - rm instantclient-basic-linux.x64-21.6.0.0.0dbru.zip && - mv "$SEQ_WORKSPACE"/.oracle/instantclient_21_6 "$SEQ_WORKSPACE"/.oracle/instantclient + wget https://download.oracle.com/otn_software/linux/instantclient/217000/instantclient-basic-linux.x64-21.7.0.0.0dbru.zip --no-check-certificate && + unzip instantclient-basic-linux.x64-21.7.0.0.0dbru.zip -d "$SEQ_WORKSPACE"/.oracle/ && + rm instantclient-basic-linux.x64-21.7.0.0.0dbru.zip && + mv "$SEQ_WORKSPACE"/.oracle/instantclient_21_7 "$SEQ_WORKSPACE"/.oracle/instantclient echo "Local Oracle instant client on Linux has been setup!" elif [[ $(uname) == 'Darwin' ]] diff --git a/dev/oracle/21-slim/privileges.sql b/dev/oracle/privileges.sql similarity index 100% rename from dev/oracle/21-slim/privileges.sql rename to dev/oracle/privileges.sql diff --git a/dev/oracle/21-slim/wait-until-healthy.sh b/dev/oracle/wait-until-healthy.sh similarity index 100% rename from dev/oracle/21-slim/wait-until-healthy.sh rename to dev/oracle/wait-until-healthy.sh diff --git a/package.json b/package.json index 0c79269fc93a..5434ee0eb43e 100644 --- a/package.json +++ b/package.json @@ -106,7 +106,7 @@ "mysql2": "^2.3.3", "node-hook": "^1.0.0", "nyc": "^15.1.0", - "oracledb": "^5.4.0", + "oracledb": "^5.5.0", "p-map": "^4.0.0", "p-props": "^4.0.0", "p-settle": "^4.1.1", @@ -244,20 +244,23 @@ "start-postgres": "bash dev/postgres/10/start.sh", "start-mssql": "bash dev/mssql/2019/start.sh", "start-db2": "bash dev/db2/11.5/start.sh", - "start-oracle": "bash dev/oracle/21-slim/start.sh", + "start-oracle-oldest": "bash dev/oracle/18-slim/start.sh", + "start-oracle-latest": "bash dev/oracle/21-slim/start.sh", "stop-mariadb": "bash dev/mariadb/10.3/stop.sh", "stop-mysql": "bash dev/mysql/5.7/stop.sh", "stop-mysql-8": "bash dev/mysql/8.0/stop.sh", "stop-postgres": "bash dev/postgres/10/stop.sh", "stop-mssql": "bash dev/mssql/2019/stop.sh", "stop-db2": "bash dev/db2/11.5/stop.sh", - "stop-oracle": "bash dev/oracle/21-slim/stop.sh", + "stop-oracle-oldest": "bash dev/oracle/18-slim/stop.sh", + "stop-oracle-latest": "bash dev/oracle/21-slim/stop.sh", "restart-mariadb": "npm run start-mariadb", "restart-mysql": "npm run start-mysql", "restart-postgres": "npm run start-postgres", "restart-mssql": "npm run start-mssql", "restart-db2": "npm run start-db2", - "restart-oracle": "npm run start-oracle", + "restart-oracle-oldest": "npm run start-oracle-oldest", + "restart-oracle-latest": "npm run start-oracle-latest", "----------------------------------------- local tests ---------------------------------------------": "", "test-unit-mariadb": "cross-env DIALECT=mariadb npm run test-unit", "test-unit-mysql": "cross-env DIALECT=mysql npm run test-unit", diff --git a/src/dialects/oracle/connection-manager.js b/src/dialects/oracle/connection-manager.js index f79c461b9142..bea8b0eb33f2 100644 --- a/src/dialects/oracle/connection-manager.js +++ b/src/dialects/oracle/connection-manager.js @@ -6,6 +6,7 @@ const AbstractConnectionManager = require('../abstract/connection-manager'); const SequelizeErrors = require('../../errors'); const parserStore = require('../parserStore')('oracle'); const { logger } = require('../../utils/logger'); +const semver = require('semver'); const debug = logger.debugContext('connection:oracle'); const DataTypes = require('../../data-types').oracle; const { promisify } = require('util'); @@ -132,6 +133,8 @@ export class OracleConnectionManager extends AbstractConnectionManager { } const connection = await this.lib.getConnection(connectionConfig); + // Setting the sequelize database version to Oracle DB server version to remove the roundtrip for DB version query + this.sequelize.options.databaseVersion = semver.coerce(connection.oracleServerVersionString).version; debug('connection acquired'); connection.on('error', error => { diff --git a/src/dialects/oracle/index.js b/src/dialects/oracle/index.js index 188474f17e56..05c128ef8326 100644 --- a/src/dialects/oracle/index.js +++ b/src/dialects/oracle/index.js @@ -54,7 +54,7 @@ OracleDialect.prototype.supports = _.merge(_.cloneDeep(AbstractDialect.prototype GEOMETRY: false }); -OracleDialect.prototype.defaultVersion = '18.4.0'; +OracleDialect.prototype.defaultVersion = '18.0.0'; OracleDialect.prototype.Query = OracleQuery; OracleDialect.prototype.queryGenerator = OracleQueryGenerator; OracleDialect.prototype.DataTypes = DataTypes; diff --git a/src/dialects/oracle/query-generator.js b/src/dialects/oracle/query-generator.js index c1d7beaa9114..ee1b5775af2b 100644 --- a/src/dialects/oracle/query-generator.js +++ b/src/dialects/oracle/query-generator.js @@ -119,7 +119,7 @@ export class OracleQueryGenerator extends AbstractQueryGenerator { } versionQuery() { - return "SELECT VERSION FROM PRODUCT_COMPONENT_VERSION WHERE PRODUCT LIKE 'Oracle%'"; + return "SELECT VERSION_FULL FROM PRODUCT_COMPONENT_VERSION WHERE PRODUCT LIKE 'Oracle%'"; } createTableQuery(tableName, attributes, options) { @@ -342,7 +342,7 @@ export class OracleQueryGenerator extends AbstractQueryGenerator { ? `WHERE (atc.OWNER = ${this.escape(schema)}) ` : 'WHERE atc.OWNER = USER ', `AND (atc.TABLE_NAME = ${this.escape(currTableName)})`, - 'ORDER BY atc.COLUMN_NAME' + 'ORDER BY atc.COLUMN_NAME, CONSTRAINT_TYPE DESC' ].join(''); } diff --git a/src/dialects/oracle/query.js b/src/dialects/oracle/query.js index 6380d3c303a0..ef9939fa325c 100644 --- a/src/dialects/oracle/query.js +++ b/src/dialects/oracle/query.js @@ -438,7 +438,7 @@ export class OracleQuery extends AbstractQuery { } else if (this.isBulkDeleteQuery()) { result = data.rowsAffected; } else if (this.isVersionQuery()) { - const version = data.rows[0].VERSION; + const version = data.rows[0].VERSION_FULL; if (version) { const versions = version.split('.'); result = `${versions[0]}.${versions[1]}.${versions[2]}`; diff --git a/test/integration/json.test.js b/test/integration/json.test.js index 039fb966a932..ca3d25e3e6c7 100644 --- a/test/integration/json.test.js +++ b/test/integration/json.test.js @@ -202,6 +202,13 @@ describe('model', () => { }); it('should be able to store strings', async function() { + if (dialect === 'oracle') { + const dbVersion = this.sequelize.options.databaseVersion; + // Oracle DB below 21c doesn't recognize a string as a valid json + if (dbVersion.localeCompare('21.0.0.0') === -1) { + this.skip(); + } + } await this.User.create({ username: 'swen', emergency_contact: 'joe' }); const user = await this.User.findOne({ where: { username: 'swen' } }); expect(user.emergency_contact).to.equal('joe'); diff --git a/yarn.lock b/yarn.lock index 5f590ae30a5a..71b743105da9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6393,10 +6393,10 @@ optionator@^0.9.1: type-check "^0.4.0" word-wrap "^1.2.3" -oracledb@^5.4.0: - version "5.4.0" - resolved "https://registry.yarnpkg.com/oracledb/-/oracledb-5.4.0.tgz#0ae79faa9a3c04d39b845b897d1e58342d275a85" - integrity sha512-JirwPg+HobTV6EuXcHoHA0n55wJiWCNHaS2TRSHLSxOB9WbDyrT8q6E5PPB7nre1V/kFCd5luHkV1z780Voyfw== +oracledb@^5.5.0: + version "5.5.0" + resolved "https://registry.yarnpkg.com/oracledb/-/oracledb-5.5.0.tgz#0cf9af5d0c0815f74849ae9ed56aee823514d71b" + integrity sha512-i5cPvMENpZP8nnqptB6l0pjiOyySj1IISkbM4Hr3yZEDdANo2eezarwZb9NQ8fTh5pRjmgpZdSyIbnn9N3AENw== ordered-read-streams@^1.0.0: version "1.0.1"