Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use separate PostgreSQL schemas for sprocs #4412

Merged
merged 30 commits into from Jul 7, 2021
Merged
Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
75aefc8
Add schema support to sql-db
mwest1066 Jun 2, 2021
5793b8b
Add random schema setting
mwest1066 Jun 2, 2021
394eb53
Fix function definitions
mwest1066 Jun 24, 2021
6173313
Fix substring() name
mwest1066 Jun 24, 2021
e0a2ff6
Change replaceAll() -> replace()
mwest1066 Jun 24, 2021
56e7a7e
Fix module.export(s)
mwest1066 Jun 24, 2021
e107ace
Change to underscores for schema name
mwest1066 Jun 24, 2021
3a4f4b1
Actually create the new schema
mwest1066 Jun 24, 2021
899005e
Merge branch 'master' into sproc-schemas
mwest1066 Jun 24, 2021
ba6048c
Fix an actual bug found by the linter!
mwest1066 Jun 24, 2021
926d1f2
Only create schema if it doesn't exist
mwest1066 Jun 29, 2021
48f7bca
Use client.escapeIdentifier() for schema
mwest1066 Jun 29, 2021
c16d704
Document use of done()
mwest1066 Jun 29, 2021
dfe3f28
Document length limit of schema prefix
mwest1066 Jun 29, 2021
eda367f
Fix setting with null schema
mwest1066 Jun 29, 2021
2f2894f
Fix schema prefix truncation
mwest1066 Jun 29, 2021
6aadf86
Fix to create non-null schema before using in search_path
mwest1066 Jun 29, 2021
581aa74
Shift new functions to end-of-file
mwest1066 Jun 29, 2021
074f37d
Force schema name to lowercase for convenience with psql
mwest1066 Jun 29, 2021
c697be2
Fix sprocs/array_and_number.sql to not check EXISTS
mwest1066 Jun 29, 2021
6c9f8c7
Rewrite query() to use queryWithClient()
mwest1066 Jun 29, 2021
645cda3
Add docs and comments about our schema system
mwest1066 Jun 29, 2021
06a1883
Merge branch 'master' into sproc-schemas
mwest1066 Jun 29, 2021
4a5e2c8
Clarify "local" in the docs
mwest1066 Jun 30, 2021
f4e254c
Rewrite type creation comment
mwest1066 Jul 1, 2021
76142cc
More comment updates about type creation
mwest1066 Jul 1, 2021
4a966d1
Change all sprocs to plain CREATE
mwest1066 Jul 1, 2021
1f2cfbe
Fix tests to actually use setRandomSearchSchema()
mwest1066 Jul 1, 2021
22b41cc
Remove sprocs/random_string.sql because migrations make it in public …
mwest1066 Jul 1, 2021
6db5561
Use more character types in setRandomSearchSchema()
mwest1066 Jul 7, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
97 changes: 89 additions & 8 deletions prairielib/lib/sql-db.js
Expand Up @@ -7,6 +7,8 @@ const { promisify } = require('util');

const error = require('./error');

let searchSchema = null;

/**
* Formats a string for debugging.
*
Expand Down Expand Up @@ -180,7 +182,65 @@ module.exports.getClient = function(callback) {
}
return ERR(err, callback); // unconditionally return
}
callback(null, client, done);
if (searchSchema != null) {
const setSearchPathSql = `SET search_path TO ${searchSchema},public`;
mwest1066 marked this conversation as resolved.
Show resolved Hide resolved
module.exports.queryWithClient(client, setSearchPathSql, {}, (err) => {
if (err) {
done(client);
return ERR(err, callback); // unconditionally return
}
callback(null, client, done);
});
} else {
callback(null, client, done);
}
});
};

/**
* Set the schema to use for the search path.
*
* @param {string} schema - The schema name to use (can be "null" to unset the search path)
* @param {(error: Error | null) => void} callback
*/
module.exports.setSearchSchema = function(schema, callback) {
searchSchema = schema;
module.exports.query('CREATE SCHEMA ' + schema, [], (err) => {
if (ERR(err, callback)) return;
callback(null);
});
};

/**
* Get the schema that is currently used for the search path.
*
* @return {string} schema in use (may be "null" to indicate no schema)
*/
module.exports.getSearchSchema = function() {
return searchSchema;
};

/**
* Generate, set, and return a random schema name.
*
* @param {string} prefix - The prefix of the new schema
* @param {(error: Error | null, schema: String) => void} callback
*/
module.exports.setRandomSearchSchema = function(prefix, callback) {
// truncated prefix (max 28 characters)
const truncPrefix = prefix.substring(30);
// 27-character timestamp in format YYYY-MM-DDTHH-MM-SS-SSSZ
const timestamp = (new Date()).toISOString().replace(/-/g, '_').replace(/:/g, '_').replace(/[.]/g, '_');
// random 6-character suffix to avoid clashes (approx 2 billion values)
const chars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ'.split('');
const suffix = _.times(6, function() {return _.sample(chars);}).join('');

// schema is guaranteed to have length at most 63 (= 28 + 1 + 27 + 1 + 6)
// which is the default PostgreSQL identifier limit
const schema = `${truncPrefix}_${timestamp}_${suffix}`;
module.exports.setSearchSchema(schema, (err) => {
if (ERR(err, callback)) return;
callback(null, schema);
});
};

Expand Down Expand Up @@ -419,14 +479,35 @@ module.exports.query = function(sql, params, callback) {
return true;
};
if (handleError(err)) return;
paramsToArray(sql, params, function(err, newSql, newParams) {
if (err) err = error.addData(err, {sql: sql, sqlParams: params});

const setSearchPath = function(callback) {
if (searchSchema != null) {
const setSearchPathSql = `SET search_path TO ${searchSchema},public`;
mwest1066 marked this conversation as resolved.
Show resolved Hide resolved
module.exports.queryWithClient(client, setSearchPathSql, {}, (err) => {
if (err) {
if (client) {
done(client);
}
return ERR(err, callback); // unconditionally return
}
callback(null);
});
} else {
callback(null);
}
};

setSearchPath(function (err) {
if (ERR(err, callback)) return;
client.query(newSql, newParams, function(err, result) {
if (handleError(err)) return;
done();
debug('query() success', 'rowCount:', result.rowCount);
callback(null, result);
paramsToArray(sql, params, function(err, newSql, newParams) {
if (err) err = error.addData(err, {sql: sql, sqlParams: params});
if (ERR(err, callback)) return;
client.query(newSql, newParams, function(err, result) {
if (handleError(err)) return;
done();
debug('query() success', 'rowCount:', result.rowCount);
callback(null, result);
});
});
});
});
Expand Down
6 changes: 6 additions & 0 deletions server.js
Expand Up @@ -1136,6 +1136,12 @@ if (config.startServer) {
callback(null);
});
},
function(callback) {
mwest1066 marked this conversation as resolved.
Show resolved Hide resolved
sqldb.setRandomSearchSchema(config.instanceId, (err) => {
if (ERR(err, callback)) return;
callback(null);
});
},
function(callback) {
sprocs.init(function(err) {
if (ERR(err, callback)) return;
Expand Down