Use separate PostgreSQL schemas for sprocs

docs/dev-guide.md

@@ -275,6 +275,31 @@ FROM
 ```
 
 
+## DB stored procedures (sprocs)
+
+* Stored procedures are created by the files in `sprocs/`. To call a stored procedure from JavaScript, use code like:
+
+```
+const workspace_id = 1342;
+const message = 'Startup successful';
+sqldb.call('workspaces_message_update', [workspace_id, message], (err, result) => {
+    if (ERR(err, callback)) return;
+    // we could use the result here if we want the return value of the stored procedure
+    callback(null);
+});
+```
+
+* The stored procedures are all contained in a separate [database schema](https://www.postgresql.org/docs/12/ddl-schemas.html) with a name like `server_2021_06_29t19_28_32_247z_e3khma`. To see a list of the schemas use the `\dn` command in `psql`.
+
+* To be able to use the stored procedures from the `psql` command line it is necessary to get the most recent schema name using `\dn` and set the `search_path` to use this schema name and the `public` schema:
+
+```
+set search_path to server_2021_06_29t19_28_32_247z_e3khma,public;
+```
+
+* During startup we initially have no non-public schema in use. We first run the migrations to update all tables in the `public` schema, then we call `sqldb.setRandomSearchSchema()` to activate a random per-execution schema, and we run the sproc creation code to generate all the stored procedures in this schema. This means that every invocation of PrairieLearn will have its own local copy of the stored procedures which are the correct versions for its code. This lets us upgrade PrairieLearn servers one at a time, while old servers are still running with their own copies of their sprocs. When PrairieLearn first starts up it has `search_path = public`, but later it will have `search_path = server_2021_06_29t19_28_32_247z_e3khma,public` so that it will first search the random schema and then fall back to `public`. The naming convention for the random schema uses the local instance name, the date, and a random string.
+
+
 ## DB schema (simplified overview)
 
 * The most important tables in the database are shown in the diagram below (also as a [PDF image](simplified-models.pdf)).

prairielib/lib/sql-db.js

@@ -7,6 +7,8 @@ const { promisify } = require('util');
 
 const error = require('./error');
 
+let searchSchema = null;
+
 /**
  * Formats a string for debugging.
  *
@@ -165,7 +167,11 @@ module.exports.close = function(callback) {
 module.exports.closeAsync = promisify(module.exports.close);
 
 /**
- * Gets a new client from the connection pool.
+ * Gets a new client from the connection pool. If `err` is not null
+ * then `client` and `done` are undefined. If `err` is null then
+ * `client` is valid and can be used. The caller MUST call
+ * `done(client)` to release the client, whether or not errors occured
+ * while using `client`.
  *
  * @param {(error: Error | null, client: import("pg").PoolClient, done: (release?: any) => void) => void} callback
  */
@@ -180,7 +186,18 @@ module.exports.getClient = function(callback) {
             }
             return ERR(err, callback); // unconditionally return
         }
-        callback(null, client, done);
+        if (searchSchema != null) {
+            const setSearchPathSql = `SET search_path TO ${client.escapeIdentifier(searchSchema)},public`;
+            module.exports.queryWithClient(client, setSearchPathSql, {}, (err) => {
+                if (err) {
+                    done(client);
+                    return ERR(err, callback); // unconditionally return
+                }
+                callback(null, client, done);
+            });
+        } else {
+            callback(null, client, done);
+        }
     });
 };
 
@@ -403,31 +420,12 @@ module.exports.endTransactionAsync = promisify(module.exports.endTransaction);
 module.exports.query = function(sql, params, callback) {
     debug('query()', 'sql:', debugString(sql));
     debug('query()', 'params:', debugParams(params));
-    if (!pool) {
-        return callback(new Error('Connection pool is not open'));
-    }
-    pool.connect(function(err, client, done) {
-        const handleError = function(err) {
-            if (!err) return false;
-            if (client) {
-                done(client);
-            }
-            const sqlError = JSON.parse(JSON.stringify(err));
-            sqlError.message = err.message;
-            err = error.addData(err, {sqlError: sqlError, sql: sql, sqlParams: params});
-            ERR(err, callback);
-            return true;
-        };
-        if (handleError(err)) return;
-        paramsToArray(sql, params, function(err, newSql, newParams) {
-            if (err) err = error.addData(err, {sql: sql, sqlParams: params});
+    module.exports.getClient((err, client, done) => {
+        if (ERR(err, callback)) return;
+        module.exports.queryWithClient(client, sql, params, (err, result) => {
+            done(client);
             if (ERR(err, callback)) return;
-            client.query(newSql, newParams, function(err, result) {
-                if (handleError(err)) return;
-                done();
-                debug('query() success', 'rowCount:', result.rowCount);
-                callback(null, result);
-            });
+            callback(null, result);
         });
     });
 };
@@ -659,3 +657,60 @@ module.exports.callWithClientZeroOrOneRow = function(client, functionName, param
  * Errors if the function returns more than one row.
  */
 module.exports.callWithClientZeroOrOneRowAsync = promisify(module.exports.callWithClientZeroOrOneRow);
+
+/**
+ * Set the schema to use for the search path.
+ *
+ * @param {string} schema - The schema name to use (can be "null" to unset the search path)
+ * @param {(error: Error | null) => void} callback
+ */
+module.exports.setSearchSchema = function(schema, callback) {
+    if (schema == null) {
+        searchSchema = schema;
+        return;
+    }
+    /* Note that as of 2021-06-29 escapeIdentifier() is undocumented. See:
+     * https://github.com/brianc/node-postgres/pull/396
+     * https://github.com/brianc/node-postgres/issues/1978
+     * https://www.postgresql.org/docs/12/sql-syntax-lexical.html
+     */
+    module.exports.query(`CREATE SCHEMA IF NOT EXISTS ${pg.Client.prototype.escapeIdentifier(schema)}`, [], (err) => {
+        if (ERR(err, callback)) return;
+        // we only set searchSchema after CREATE to avoid the above query() call using searchSchema
+        searchSchema = schema;
+        callback(null);
+    });
+};
+
+/**
+ * Get the schema that is currently used for the search path.
+ *
+ * @return {string} schema in use (may be "null" to indicate no schema)
+ */
+module.exports.getSearchSchema = function() {
+    return searchSchema;
+};
+
+/**
+ * Generate, set, and return a random schema name.
+ *
+ * @param {string} prefix - The prefix of the new schema, only the first 28 characters will be used (after lowercasing).
+ * @param {(error: Error | null, schema: String) => void} callback
+ */
+module.exports.setRandomSearchSchema = function(prefix, callback) {
+    // truncated prefix (max 28 characters)
+    const truncPrefix = prefix.substring(0, 28);
+    // 27-character timestamp in format YYYY-MM-DDTHH-MM-SS-SSSZ
+    const timestamp = (new Date()).toISOString().replace(/-/g, '_').replace(/:/g, '_').replace(/[.]/g, '_');
+    // random 6-character suffix to avoid clashes (approx 2 billion values)
+    const chars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ'.split('');
+    const suffix = _.times(6, function() {return _.sample(chars);}).join('');
+
+    // schema is guaranteed to have length at most 63 (= 28 + 1 + 27 + 1 + 6)
+    // which is the default PostgreSQL identifier limit
+    const schema = `${truncPrefix}_${timestamp}_${suffix}`.toLowerCase();
+    module.exports.setSearchSchema(schema, (err) => {
+        if (ERR(err, callback)) return;
+        callback(null, schema);
+    });
+};

server.js

@@ -1136,6 +1136,21 @@ if (config.startServer) {
                 callback(null);
             });
         },
+        function(callback) {
+            // We create and activate a random DB schema name
+            // (https://www.postgresql.org/docs/12/ddl-schemas.html)
+            // after we have run the migrations but before we create
+            // the sprocs. This means all tables (from migrations) are
+            // in the public schema, but all sprocs are in the random
+            // schema. Every server invocation thus has its own copy
+            // of its sprocs, allowing us to update servers while old
+            // servers are still running. See docs/dev-guide.md for
+            // more info.
+            sqldb.setRandomSearchSchema(config.instanceId, (err) => {
+                if (ERR(err, callback)) return;
+                callback(null);
+            });
+        },
         function(callback) {
             sprocs.init(function(err) {
                 if (ERR(err, callback)) return;

sprocs/array_and_number.sql

@@ -1,7 +1,6 @@
---create types
-DO $$
-BEGIN
-    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'array_and_number') THEN
-        CREATE TYPE array_and_number AS (arr DOUBLE PRECISION[], number INTEGER);
-    END IF;
-END$$;
+-- We do not test whether the type already exists.
+-- This is because we will be creating it within the current default schema, which will be empty.
+-- We could test whether the type exists in any schema by querying pg_type like in migrations/000_initial_state.sql.
+-- But that would test for the type in any schema, and we only care about schema on the search path.
+-- It's hard to restrict the "does this type exist" query to just the current search path.
+CREATE TYPE array_and_number AS (arr DOUBLE PRECISION[], number INTEGER);