Redeclaration of virtual path in Schema, makes it stop populating

[ezesalmon]

Prerequisites

• I have written a descriptive issue title
• I have searched existing issues to ensure the bug has not already been reported

Mongoose version

6.10.0

Node.js version

18.13.0

MongoDB server version

6.0

Typescript version (if applicable)

No response

Description

I'm facing a trouble in my current project.
I allow the user to create virtual path on its own, but if I want to allow the user to change the options on the virtual path that he already created, the path stops populating.

Removing the virtual path by using Schema.removeVirtual does not change the behaviour.

I've tried to debug the library but couldn't reach any conclusion at all 👎

Steps to Reproduce

The code below contains everything that is needed to reproduce the behaviour.
There are some NOTEx marks on the code.
I think it's pretty easy to follow.

'use strict';
const mongoose = require('mongoose');

const personSch = new mongoose.Schema(
    {
        firstName: { type: mongoose.SchemaTypes.String, required: true },
        surname: { type: mongoose.SchemaTypes.String, trim: true },
        nat: { type: mongoose.SchemaTypes.String, required: true, uppercase: true, minLength: 2, maxLength: 2 }
    },
    { strict: true, timestamps: true }
);
// personSch.virtual('nationality', {
//     localField: 'nat',
//     foreignField: 'key',
//     ref: 'Nat',
//     justOne: true
// });
let Person = mongoose.model('Person', personSch);

const natSch = new mongoose.Schema(
    {
        key: { type: mongoose.SchemaTypes.String, uppercase: true, index: true, minLength: 2, maxLength: 2 },
        desc: { type: mongoose.SchemaTypes.String, trim: true }
    },
    { strict: true }
);
let Nat = mongoose.model('Nat', natSch);

function listModels(dbConn) {
    let models = dbConn.connection.models;
    for (let k in models) {
        let m = models[k];
        if (k === 'Person') {
            console.log();
            const modelPaths = Object.keys(m.schema.paths);
            console.log(m.modelName, 'paths:', modelPaths);
            const modelVirtuals = Object.keys(m.schema.virtuals);
            console.log(m.modelName, 'virtuals:', modelVirtuals);
            console.log();
        }
    }
}

async function listPeople(populates) {
    let peopleList;
    if (populates) peopleList = await Person.find().populate(populates);
    else peopleList = await Person.find();

    for (const person of peopleList) {
        console.log(person.firstName, person.nat, person.nat2, person[populates] ? person[populates].desc : 'n/a');
    }
}

async function test() {
    console.log('Connecting...');
    mongoose.set('strictQuery', false);
    // mongoose.set('debug', true);

    const connectionString = 'mongodb://127.0.0.1/dbtesting';
    const dbConn = await mongoose.connect(connectionString, { useNewUrlParser: true, useUnifiedTopology: true });
    console.log('Connected!!!', 'Mongoose v' + dbConn.version);
    // listModels(dbConn);
    await Person.deleteMany();
    await Nat.deleteMany();

    // /////////////////////////////////////////////////
    // DOCUMENT CREATION
    // /////////////////////////////////////////////////
    try {
        let n = new Nat({ key: 'ES', desc: 'Spain' });
        await n.save();
        n = new Nat({ key: 'IT', desc: 'Italy' });
        await n.save();
        n = new Nat({ key: 'FR', desc: 'French' });
        await n.save();

        let p = new Person({ firstName: 'Pepe', surname: 'Pérez', nat: 'it' });
        await p.save();
        p = new Person({ firstName: 'Paco', surname: 'Matinez', nat: 'es' });
        await p.save();
        p = new Person({ firstName: 'John', surname: 'Doe', nat: 'us' });
        await p.save();
    } catch (error) {
        console.error(error.message);
    }

    await listPeople();

    console.warn('adding virtual!');
    // Person.schema.removeVirtual('nationality');
    // listModels(dbConn);
    Person.schema.virtual('nationality', {
        localField: 'nat',
        foreignField: 'key',
        ref: 'Nat',
        justOne: true
    });
    await listPeople('nationality');

    console.warn('Adding virtual 2!');
    Person.schema.virtual('nationalityBis', {
        localField: 'nat',
        foreignField: 'key',
        ref: 'Nat',
        justOne: true
    });
    await listPeople('nationality');
    await listPeople('nationalityBis');
    // [NOTE1: Until this line everything is working as expected :-) ]

    // [NOTE2: Add a 2nd nat field to the schema, and its vitual ]
    const newPersonSchema = Person.schema.clone();
    newPersonSchema.add(new mongoose.Schema({ nat2: { type: String, uppercase: true, minLength: 2, maxLength: 2 } }));

    // [NOTE3: Removing or not the virtual ends with the same result ]
    newPersonSchema.removeVirtual('nationality');
    newPersonSchema.virtual('nationality', {
        localField: 'nat2',
        foreignField: 'key',
        ref: 'Nat',
        justOne: true
    });
    //
    newPersonSchema.virtual('nationality2', {
        localField: 'nat2',
        foreignField: 'key',
        ref: 'Nat',
        justOne: true
    });

    mongoose.deleteModel('Person');
    Person = mongoose.model('Person', newPersonSchema);
    listModels(dbConn);
    let peopleList = await Person.find();
    for (const person of peopleList) {
        person.nat2 = 'FR';
        await person.save();
    }

    // [NOTE4: Virtual field is not there!!!!!!! ]
    await listPeople('nationality');
    console.log();
    await listPeople('nationalityBis');
    console.log();
    await listPeople('nationality2');
    console.log();

    console.log('Finished!');
}

test().then(() => process.exit());

Expected Behavior

I expect the redefining a virtual path on the schema, does not break the virtual path at all.
Or have a workaround to avoid this issue if I want dynamically change the virtual path during the execution of my program.

Thanks so much for have a look at this issue!

[IslandRhythms]

If my understanding of your issue is correct, I cannot reproduce. Every doc has FR set after note 4.

'use strict';
const mongoose = require('mongoose');

const personSch = new mongoose.Schema(
    {
        firstName: { type: mongoose.SchemaTypes.String, required: true },
        surname: { type: mongoose.SchemaTypes.String, trim: true },
        nat: { type: mongoose.SchemaTypes.String, required: true, uppercase: true, minLength: 2, maxLength: 2 }
    },
    { strict: true, timestamps: true }
);
// personSch.virtual('nationality', {
//     localField: 'nat',
//     foreignField: 'key',
//     ref: 'Nat',
//     justOne: true
// });
let Person = mongoose.model('Person', personSch);

const natSch = new mongoose.Schema(
    {
        key: { type: mongoose.SchemaTypes.String, uppercase: true, index: true, minLength: 2, maxLength: 2 },
        desc: { type: mongoose.SchemaTypes.String, trim: true }
    },
    { strict: true }
);
let Nat = mongoose.model('Nat', natSch);

function listModels(dbConn) {
    let models = dbConn.connection.models;
    for (let k in models) {
        let m = models[k];
        if (k === 'Person') {
            console.log();
            const modelPaths = Object.keys(m.schema.paths);
            console.log(m.modelName, 'paths:', modelPaths);
            const modelVirtuals = Object.keys(m.schema.virtuals);
            console.log(m.modelName, 'virtuals:', modelVirtuals);
            console.log();
        }
    }
}

async function listPeople(populates) {
    let peopleList;
    if (populates) peopleList = await Person.find().populate(populates);
    else peopleList = await Person.find();

    for (const person of peopleList) {
        console.log(person.firstName, person.nat, person.nat2, person[populates] ? person[populates].desc : 'n/a');
    }
}

async function test() {
    console.log('Connecting...');
    mongoose.set('strictQuery', false);
    // mongoose.set('debug', true);

    const connectionString = 'mongodb://127.0.0.1/dbtesting';
    const dbConn = await mongoose.connect(connectionString, { useNewUrlParser: true, useUnifiedTopology: true });
    console.log('Connected!!!', 'Mongoose v' + dbConn.version);
    // listModels(dbConn);
    await Person.deleteMany();
    await Nat.deleteMany();

    // /////////////////////////////////////////////////
    // DOCUMENT CREATION
    // /////////////////////////////////////////////////
    try {
        let n = new Nat({ key: 'ES', desc: 'Spain' });
        await n.save();
        n = new Nat({ key: 'IT', desc: 'Italy' });
        await n.save();
        n = new Nat({ key: 'FR', desc: 'French' });
        await n.save();

        let p = new Person({ firstName: 'Pepe', surname: 'Pérez', nat: 'it' });
        await p.save();
        p = new Person({ firstName: 'Paco', surname: 'Matinez', nat: 'es' });
        await p.save();
        p = new Person({ firstName: 'John', surname: 'Doe', nat: 'us' });
        await p.save();
    } catch (error) {
        console.error(error.message);
    }

    await listPeople();

    console.warn('adding virtual!');
    // Person.schema.removeVirtual('nationality');
    // listModels(dbConn);
    Person.schema.virtual('nationality', {
        localField: 'nat',
        foreignField: 'key',
        ref: 'Nat',
        justOne: true
    });
    await listPeople('nationality');

    console.warn('Adding virtual 2!');
    Person.schema.virtual('nationalityBis', {
        localField: 'nat',
        foreignField: 'key',
        ref: 'Nat',
        justOne: true
    });
    await listPeople('nationality');
    await listPeople('nationalityBis');
    // [NOTE1: Until this line everything is working as expected :-) ]

    // [NOTE2: Add a 2nd nat field to the schema, and its vitual ]
    const newPersonSchema = Person.schema.clone();
    newPersonSchema.add(new mongoose.Schema({ nat2: { type: String, uppercase: true, minLength: 2, maxLength: 2 } }));

    // [NOTE3: Removing or not the virtual ends with the same result ]
    newPersonSchema.removeVirtual('nationality');
    newPersonSchema.virtual('nationality', {
        localField: 'nat2',
        foreignField: 'key',
        ref: 'Nat',
        justOne: true
    });
    //
    newPersonSchema.virtual('nationality2', {
        localField: 'nat2',
        foreignField: 'key',
        ref: 'Nat',
        justOne: true
    });

    mongoose.deleteModel('Person');
    Person = mongoose.model('Person', newPersonSchema);
    listModels(dbConn);
    let peopleList = await Person.find();
    for (const person of peopleList) {
        person.nat2 = 'FR';
        await person.save();
    }

    // [NOTE4: Virtual field is not there!!!!!!! ]
    console.log('after note 4')
    await listPeople('nationality');
    console.log();
    await listPeople('nationalityBis');
    console.log();
    await listPeople('nationality2');
    console.log();

    console.log('Finished!');
}

test().then(() => process.exit());

output:

Connecting...
Connected!!! Mongoose v6.10.0
Pepe IT undefined n/a
Paco ES undefined n/a
John US undefined n/a
adding virtual!
Pepe IT undefined Italy
Paco ES undefined Spain
John US undefined n/a
Adding virtual 2!
Pepe IT undefined Italy
Paco ES undefined Spain
John US undefined n/a
Pepe IT undefined Italy
Paco ES undefined Spain
John US undefined n/a

Person paths: [
  'firstName', 'surname',
  'nat',       '_id',
  'createdAt', 'updatedAt',
  '__v',       'nat2'
]
Person virtuals: [ 'id', 'nationalityBis', 'nationality', 'nationality2' ]

after note 4
Pepe IT FR n/a
Paco ES FR n/a
John US FR n/a

Pepe IT FR Italy
Paco ES FR Spain
John US FR n/a

Pepe IT FR French
Paco ES FR French
John US FR French

Finished!

[ezesalmon]

There are 3 virtuals paths declared:

1. nationality
2. nationality2
3. nationalityBis

I did this to check that adding new virtual paths, works OK.

This is the result of populating nationality

Pepe IT FR n/a
Paco ES FR n/a
John US FR n/a

RESULT: populated field is not there. THIS IS AN ERROR
nationality was the only virtual path deleted and redeclared
As mentioned before, removing or not the virtual path by using Schema.removeVirtual does not change the behaviour.

This is the result of populating nationality2

Pepe IT FR Italy
Paco ES FR Spain
John US FR n/a

Result: OK!
nationality2 was declared once

This is the result of populating nationalityBis

Pepe IT FR French
Paco ES FR French
John US FR French

Result: OK!
nationalityBis was declared once

So, the error is on the nationality virtual path.
The other virtual paths are there to check that adding virtuals dynamically to the schema WORKS fine, but when redeclaring the same virtual path, stops working.
Apologies for the misleading example.

Thanks again!

[ezesalmon]

I don't know if it is any help, but I've prune the script a bit just to see the issue better. Thanks.

'use strict';
const mongoose = require('mongoose');

const personSch = new mongoose.Schema(
    {
        firstName: { type: mongoose.SchemaTypes.String, required: true },
        surname: { type: mongoose.SchemaTypes.String, trim: true },
        nat: { type: mongoose.SchemaTypes.String, required: true, uppercase: true, minLength: 2, maxLength: 2 }
    },
    { strict: true, timestamps: true }
);
personSch.virtual('nationality', {
    localField: 'nat',
    foreignField: 'key',
    ref: 'Nat',
    justOne: true
});
let Person = mongoose.model('Person', personSch);

const natSch = new mongoose.Schema(
    {
        key: { type: mongoose.SchemaTypes.String, uppercase: true, index: true, minLength: 2, maxLength: 2 },
        desc: { type: mongoose.SchemaTypes.String, trim: true }
    },
    { strict: true }
);
let Nat = mongoose.model('Nat', natSch);

async function listPeople(populates) {
    let peopleList;
    if (populates) peopleList = await Person.find().populate(populates);
    else peopleList = await Person.find();

    for (const person of peopleList) {
        console.log(person.firstName, person.nat, person[populates] ? person[populates].desc : 'n/a');
    }
}

async function test() {
    console.log('Connecting...');
    mongoose.set('strictQuery', false);
    // mongoose.set('debug', true);

    const connectionString = 'mongodb://127.0.0.1/dbtesting';
    const dbConn = await mongoose.connect(connectionString, { useNewUrlParser: true, useUnifiedTopology: true });
    console.log('Connected!!!', 'Mongoose v' + dbConn.version);

    await Person.deleteMany();
    await Nat.deleteMany();

    // /////////////////////////////////////////////////
    // DOCUMENT CREATION
    // /////////////////////////////////////////////////
    try {
        let n = new Nat({ key: 'ES', desc: 'Spain' });
        await n.save();
        n = new Nat({ key: 'IT', desc: 'Italy' });
        await n.save();
        n = new Nat({ key: 'FR', desc: 'French' });
        await n.save();

        let p = new Person({ firstName: 'Pepe', surname: 'Pérez', nat: 'it' });
        await p.save();
        p = new Person({ firstName: 'Paco', surname: 'Matinez', nat: 'es' });
        await p.save();
        p = new Person({ firstName: 'John', surname: 'Doe', nat: 'us' });
        await p.save();
    } catch (error) {
        console.error(error.message);
    }

    await listPeople('nationality');

    console.warn('re adding virtual!');
    Person.schema.removeVirtual('nationality');
    // listModels(dbConn);
    Person.schema.virtual('nationality', {
        localField: 'nat',
        foreignField: 'key',
        ref: 'Nat',
        justOne: true
    });
    await listPeople('nationality');

    console.log('Finished!');
}

test().then(() => process.exit());

This is the output

[vkarpov15]

The problem here is that Mongoose doesn't support modifying a model's schema after you've compiled the model in general. In this case, the issue is that Person.schema.virtual('nationality', { ... }) tries to add some pre('init') hooks on the model, but those changes don't get picked up.

What are you trying to achieve by redefining the virtual after model compilation? You can always add additional options to populate() as follows, no need to recreate the virtual just to change options.

async function listPeople(populates) {
    let peopleList;
    // Populate with options
    if (populates) peopleList = await Person.find().populate({ path: populates, match: { desc: 'Spain' } });
    else peopleList = await Person.find();

    for (const person of peopleList) {
        console.log(person.firstName, person.nat, person[populates] ? person[populates].desc : 'n/a');
    }
}

[github-actions]

This issue is stale because it has been open 14 days with no activity. Remove stale label or comment or this will be closed in 5 days

[ezesalmon]

(Sorry, I wat offline for a while and I couldn't answer).

What are you trying to achieve by redefining the virtual after model compilation?

A user in the system we have developed has the ability to create and destroy fields on the database. Those actions are reflected by modifying mongoose schemas. And those actions includes create and delete virtual fields.
We do recompile and redeclare models, because we detected it was needed.

Is there any way to clone a schema, and remove some virtual fields? And then use that schema to redeclares/recompile the model? If the answer is yes, can you please explain how?
Because if we can achieve to do that, our problem will disappear.

Thanks in advance.

[vkarpov15]

We found a minor bug with removeVirtual() that we fixed in #13255. With that fix, the following script works. Notice that the following script recreates the Person model, rather than modifying Person.schema.

'use strict';
const mongoose = require('mongoose');

const personSch = new mongoose.Schema(
    {
        firstName: { type: mongoose.SchemaTypes.String, required: true },
        surname: { type: mongoose.SchemaTypes.String, trim: true },
        nat: { type: mongoose.SchemaTypes.String, required: true, uppercase: true, minLength: 2, maxLength: 2 }
    },
    { strict: true, timestamps: true }
);
personSch.virtual('nationality', {
    localField: 'nat',
    foreignField: 'key',
    ref: 'Nat',
    justOne: true
});
let Person = mongoose.model('Person', personSch.clone(), 'people');

const natSch = new mongoose.Schema(
    {
        key: { type: mongoose.SchemaTypes.String, uppercase: true, index: true, minLength: 2, maxLength: 2 },
        desc: { type: mongoose.SchemaTypes.String, trim: true }
    },
    { strict: true }
);
let Nat = mongoose.model('Nat', natSch);

async function listPeople(populates) {
    let peopleList;
    if (populates) peopleList = await Person.find().populate({ path: populates, match: { desc: 'Spain' } });
    else peopleList = await Person.find();

    for (const person of peopleList) {
        console.log(person.firstName, person.nat, person[populates] ? person[populates].desc : 'n/a');
    }
}

async function test() {
    console.log('Connecting...');
    mongoose.set('strictQuery', false);
    // mongoose.set('debug', true);

    const connectionString = 'mongodb://127.0.0.1/dbtesting';
    const dbConn = await mongoose.connect(connectionString, { useNewUrlParser: true, useUnifiedTopology: true });
    console.log('Connected!!!', 'Mongoose v' + dbConn.version);

    await Person.deleteMany();
    await Nat.deleteMany();

    // /////////////////////////////////////////////////
    // DOCUMENT CREATION
    // /////////////////////////////////////////////////
    try {
        let n = new Nat({ key: 'ES', desc: 'Spain' });
        await n.save();
        n = new Nat({ key: 'IT', desc: 'Italy' });
        await n.save();
        n = new Nat({ key: 'FR', desc: 'French' });
        await n.save();

        let p = new Person({ firstName: 'Pepe', surname: 'Pérez', nat: 'it' });
        await p.save();
        p = new Person({ firstName: 'Paco', surname: 'Matinez', nat: 'es' });
        await p.save();
        p = new Person({ firstName: 'John', surname: 'Doe', nat: 'us' });
        await p.save();
    } catch (error) {
        console.error(error.message);
    }

    await listPeople('nationality');

    console.warn('re adding virtual!');
    personSch.removeVirtual('nationality');
    // listModels(dbConn);
    personSch.virtual('nationality', {
        localField: 'nat',
        foreignField: 'key',
        ref: 'Nat',
        justOne: true
    });
    Person = mongoose.model('Person', personSch.clone(), 'people', { overwriteModels: true });
    await listPeople('nationality');

    console.log('Finished!');
}

test().then(() => process.exit());

The removeVirtual() fix will be released in v7.0.4. Have you upgraded to Mongoose 7 yet, or do you need this fix backported to Mongoose 6?

[ezesalmon]

Hello!!!
Many thanks for fixing the issue with the removeVirtual method. This is good enough to correct the issue in our application! 😄

Unfortunately, we are still using Mongoose 6.x! And we are not planning to upgrade to Mongoose 7 until summer.
So, if you can backport the fix to Mongoose 6 will be amazing for us!

Many thanks!!!!