fix(embeddables): support partial loading hints

Closes #3673

packages/knex/src/AbstractSqlDriver.ts

@@ -863,6 +863,53 @@ export abstract class AbstractSqlDriver<C extends AbstractSqlConnection = Abstra
     return orderBy;
   }
 
+  protected normalizeFields<T extends object>(fields: Field<T>[], prefix = ''): string[] {
+    const ret: string[] = [];
+
+    for (const field of fields) {
+      if (typeof field === 'string') {
+        ret.push(prefix + field);
+        continue;
+      }
+
+      if (Utils.isPlainObject(field)) {
+        for (const key of Object.keys(field)) {
+          ret.push(...this.normalizeFields(field[key], key + '.'));
+        }
+      }
+    }
+
+    return ret;
+  }
+
+  protected processField<T extends object>(meta: EntityMetadata<T>, prop: EntityProperty<T> | undefined, field: string, ret: Field<T>[], populate: PopulateOptions<T>[], joinedProps: PopulateOptions<T>[], qb: QueryBuilder<T>): void {
+    if (!prop || (prop.reference === ReferenceType.ONE_TO_ONE && !prop.owner)) {
+      return;
+    }
+
+    if (prop.reference === ReferenceType.EMBEDDED) {
+      if (prop.object) {
+        ret.push(prop.fieldNames[0]);
+        return;
+      }
+
+      const parts = field.split('.');
+      const top = parts.shift();
+
+      for (const key of Object.keys(prop.embeddedProps)) {
+        if (!top || key === top) {
+          this.processField(meta, prop.embeddedProps[key], parts.join('.'), ret, populate, joinedProps, qb);
+        }
+      }
+
+      return;
+    }
+
+    if (prop.fieldNames) {
+      ret.push(prop.fieldNames[0]);
+    }
+  }
+
   protected buildFields<T extends object>(meta: EntityMetadata<T>, populate: PopulateOptions<T>[], joinedProps: PopulateOptions<T>[], qb: QueryBuilder<T>, fields?: Field<T>[]): Field<T>[] {
     const lazyProps = meta.props.filter(prop => prop.lazy && !populate.some(p => p.field === prop.name || p.all));
     const hasLazyFormulas = meta.props.some(p => p.lazy && p.formula);
@@ -873,43 +920,23 @@ export abstract class AbstractSqlDriver<C extends AbstractSqlConnection = Abstra
     if (joinedProps.length > 0) {
       ret.push(...this.getFieldsForJoinedLoad(qb, meta, fields, populate));
     } else if (fields) {
-      for (const field of [...fields]) {
-        if (field.toString().includes('.')) {
-          const parts = fields.toString().split('.');
-          const rootPropName = parts.shift()!; // first one is the `prop`
-          const prop = QueryHelper.findProperty(rootPropName, {
-            metadata: this.metadata,
-            platform: this.platform,
-            entityName: meta.className,
-            where: {},
-            aliasMap: qb.getAliasMap(),
-          });
-
-          if (prop?.reference === ReferenceType.EMBEDDED) {
-            const nest = (p: EntityProperty): EntityProperty => parts.length > 0 ? nest(p.embeddedProps[parts.shift()!]) : p;
-            const childProp = nest(prop);
-            ret.push(childProp.fieldNames[0]);
-            continue;
-          }
-        }
-
-        if (Utils.isPlainObject(field) || field.toString().includes('.')) {
+      for (const field of this.normalizeFields(fields)) {
+        if (field === '*') {
+          ret.push('*');
           continue;
         }
 
-        const prop = QueryHelper.findProperty(field.toString(), {
+        const parts = field.split('.');
+        const rootPropName = parts.shift()!; // first one is the `prop`
+        const prop = QueryHelper.findProperty<T>(rootPropName, {
           metadata: this.metadata,
           platform: this.platform,
           entityName: meta.className,
-          where: {},
+          where: {} as FilterQuery<T>,
           aliasMap: qb.getAliasMap(),
         });
 
-        if (prop?.reference === ReferenceType.ONE_TO_ONE && !prop.owner) {
-          continue;
-        }
-
-        ret.push(field);
+        this.processField(meta, prop, parts.join('.'), ret, populate, joinedProps, qb);
       }
 
       ret.unshift(...meta.primaryKeys.filter(pk => !fields.includes(pk)));
@@ -934,7 +961,7 @@ export abstract class AbstractSqlDriver<C extends AbstractSqlConnection = Abstra
         .forEach(prop => ret.push(prop.name));
     }
 
-    return ret.length > 0 ? ret : ['*'];
+    return ret.length > 0 ? Utils.unique(ret) : ['*'];
   }
 
 }

tests/EntityManager.mysql.test.ts

@@ -2201,7 +2201,7 @@ describe('EntityManagerMySql', () => {
     await orm.em.findOneOrFail(FooBar2, { random: { $gt: 0.5 } }, { having: { random: { $gt: 0.5 } } });
     expect(mock.mock.calls[1][0]).toMatch('begin');
     expect(mock.mock.calls[2][0]).toMatch('insert into `foo_bar2` (`name`) values (?)');
-    expect(mock.mock.calls[3][0]).toMatch('select `f0`.`id`, `f0`.`version`, `f0`.`version` from `foo_bar2` as `f0` where `f0`.`id` in (?)');
+    expect(mock.mock.calls[3][0]).toMatch('select `f0`.`id`, `f0`.`version` from `foo_bar2` as `f0` where `f0`.`id` in (?)');
     expect(mock.mock.calls[4][0]).toMatch('commit');
     expect(mock.mock.calls[5][0]).toMatch('select `f0`.*, (select 123) as `random` from `foo_bar2` as `f0` where (select 123) > ? having (select 123) > ? limit ?');
   });

tests/EntityManager.postgre.test.ts

@@ -781,7 +781,7 @@ describe('EntityManagerPostgre', () => {
     });
     expect(mock.mock.calls.length).toBe(3);
     expect(mock.mock.calls[0][0]).toMatch('begin');
-    expect(mock.mock.calls[1][0]).toMatch('select "b0"."uuid_pk", "b0"."created_at", "b0"."title", "b0"."price", "b0".price * 1.19 as "price_taxed", "b0"."double", "b0"."meta", "b0"."author_id", "b0"."publisher_id", "a1"."id" as "a1__id", "a1"."created_at" as "a1__created_at", "a1"."updated_at" as "a1__updated_at", "a1"."name" as "a1__name", "a1"."email" as "a1__email", "a1"."age" as "a1__age", "a1"."terms_accepted" as "a1__terms_accepted", "a1"."optional" as "a1__optional", "a1"."identities" as "a1__identities", "a1"."born" as "a1__born", "a1"."born_time" as "a1__born_time", "a1"."favourite_book_uuid_pk" as "a1__favourite_book_uuid_pk", "a1"."favourite_author_id" as "a1__favourite_author_id", "b0".price * 1.19 as "price_taxed" from "book2" as "b0" left join "author2" as "a1" on "b0"."author_id" = "a1"."id" where "b0"."author_id" is not null for update of "b0" skip locked');
+    expect(mock.mock.calls[1][0]).toMatch('select "b0"."uuid_pk", "b0"."created_at", "b0"."title", "b0"."price", "b0".price * 1.19 as "price_taxed", "b0"."double", "b0"."meta", "b0"."author_id", "b0"."publisher_id", "a1"."id" as "a1__id", "a1"."created_at" as "a1__created_at", "a1"."updated_at" as "a1__updated_at", "a1"."name" as "a1__name", "a1"."email" as "a1__email", "a1"."age" as "a1__age", "a1"."terms_accepted" as "a1__terms_accepted", "a1"."optional" as "a1__optional", "a1"."identities" as "a1__identities", "a1"."born" as "a1__born", "a1"."born_time" as "a1__born_time", "a1"."favourite_book_uuid_pk" as "a1__favourite_book_uuid_pk", "a1"."favourite_author_id" as "a1__favourite_author_id" from "book2" as "b0" left join "author2" as "a1" on "b0"."author_id" = "a1"."id" where "b0"."author_id" is not null for update of "b0" skip locked');
     expect(mock.mock.calls[2][0]).toMatch('commit');
   });
 

tests/features/embeddables/embedded-entities.postgres.test.ts

@@ -267,6 +267,20 @@ describe('embedded entities in postgresql', () => {
     expect(mock.mock.calls[11][0]).toMatch('select "u0".* from "user" as "u0" where ("u0"."address4"->>\'number\')::float8 > $1 limit $2');
   });
 
+  test('partial loading', async () => {
+    const user = createUser();
+    await orm.em.persistAndFlush(user);
+    orm.em.clear();
+
+    const mock = mockLogger(orm, ['query']);
+    await orm.em.fork().find(User, {}, { fields: ['address2'] });
+    await orm.em.fork().find(User, {}, { fields: [{ address2: ['street', 'city'] }] });
+    await orm.em.fork().find(User, {}, { fields: ['address2.street', 'address2.city'] });
+    expect(mock.mock.calls[0][0]).toMatch('select "u0"."id", "u0"."addr_street", "u0"."addr_postal_code", "u0"."addr_city", "u0"."addr_country" from "user" as "u0"');
+    expect(mock.mock.calls[1][0]).toMatch('select "u0"."id", "u0"."addr_street", "u0"."addr_city" from "user" as "u0"');
+    expect(mock.mock.calls[2][0]).toMatch('select "u0"."id", "u0"."addr_street", "u0"."addr_city" from "user" as "u0"');
+  });
+
   test('partial loading', async () => {
     const mock = mockLogger(orm, ['query']);
 

tests/features/embeddables/nested-embeddables.postgres.test.ts

@@ -266,6 +266,56 @@ describe('embedded entities in postgres', () => {
     await expect(orm.em.findOneOrFail(User, { profile2: { identity: { city: 'London 1' } as any } })).rejects.toThrowError(err2);
   });
 
+  test('partial loading', async () => {
+    const user1 = new User();
+    user1.name = 'Uwe';
+    user1.profile1 = new Profile('u1', new Identity('e1', new IdentityMeta('f1', 'b1')));
+    user1.profile2 = new Profile('u2', new Identity('e2', new IdentityMeta('f2', 'b2')));
+
+    const user2 = new User();
+    user2.name = 'Uschi';
+    user2.profile1 = new Profile('u3', new Identity('e3'));
+    user2.profile1.identity.links.push(new IdentityLink('l1'), new IdentityLink('l2'));
+    user2.profile2 = new Profile('u4', new Identity('e4', new IdentityMeta('f4')));
+    user2.profile2.identity.links.push(new IdentityLink('l3'), new IdentityLink('l4'));
+
+    await orm.em.persistAndFlush([user1, user2]);
+    orm.em.clear();
+
+    const mock = mockLogger(orm, ['query']);
+
+    await orm.em.fork().find(User, {}, { fields: ['profile1'] });
+    await orm.em.fork().find(User, {}, { fields: ['profile2'] });
+
+    await orm.em.fork().find(User, {}, { fields: ['profile1.username'] });
+    await orm.em.fork().find(User, {}, { fields: ['profile2.username'] });
+
+    await orm.em.fork().find(User, {}, { fields: ['profile1.identity'] });
+    await orm.em.fork().find(User, {}, { fields: ['profile2.identity'] });
+
+    await orm.em.fork().find(User, {}, { fields: ['profile1.identity.email'] });
+    await orm.em.fork().find(User, {}, { fields: ['profile2.identity.email'] });
+
+    await orm.em.fork().find(User, {}, { fields: ['profile1.identity.meta.foo'] });
+    await orm.em.fork().find(User, {}, { fields: ['profile2.identity.meta.foo'] });
+
+    await orm.em.fork().find(User, {}, { fields: [{ profile1: ['identity'] }] });
+    await orm.em.fork().find(User, {}, { fields: [{ profile2: ['identity'] }] });
+
+    expect(mock.mock.calls[0][0]).toMatch('select "u0"."id", "u0"."profile1_username", "u0"."profile1_identity_email", "u0"."profile1_identity_meta_foo", "u0"."profile1_identity_meta_bar", "u0"."profile1_identity_links" from "user" as "u0"');
+    expect(mock.mock.calls[1][0]).toMatch('select "u0"."id", "u0"."profile2" from "user" as "u0"');
+    expect(mock.mock.calls[2][0]).toMatch('select "u0"."id", "u0"."profile1_username" from "user" as "u0"');
+    expect(mock.mock.calls[3][0]).toMatch('select "u0"."id", "u0"."profile2" from "user" as "u0"');
+    expect(mock.mock.calls[4][0]).toMatch('select "u0"."id", "u0"."profile1_identity_email", "u0"."profile1_identity_meta_foo", "u0"."profile1_identity_meta_bar", "u0"."profile1_identity_links" from "user" as "u0"');
+    expect(mock.mock.calls[5][0]).toMatch('select "u0"."id", "u0"."profile2" from "user" as "u0"');
+    expect(mock.mock.calls[6][0]).toMatch('select "u0"."id", "u0"."profile1_identity_email" from "user" as "u0"');
+    expect(mock.mock.calls[7][0]).toMatch('select "u0"."id", "u0"."profile2" from "user" as "u0"');
+    expect(mock.mock.calls[8][0]).toMatch('select "u0"."id", "u0"."profile1_identity_meta_foo" from "user" as "u0"');
+    expect(mock.mock.calls[9][0]).toMatch('select "u0"."id", "u0"."profile2" from "user" as "u0"');
+    expect(mock.mock.calls[10][0]).toMatch('select "u0"."id", "u0"."profile1_identity_email", "u0"."profile1_identity_meta_foo", "u0"."profile1_identity_meta_bar", "u0"."profile1_identity_links" from "user" as "u0"');
+    expect(mock.mock.calls[11][0]).toMatch('select "u0"."id", "u0"."profile2" from "user" as "u0"');
+  });
+
   test('#assign() works with nested embeddables', async () => {
     const jon = new User();
 

tests/features/joined-strategy.postgre.test.ts

@@ -229,7 +229,7 @@ describe('Joined loading strategy', () => {
     const books = await orm.em.find(Book2, {}, { populate: ['tags'], strategy: LoadStrategy.JOINED, orderBy: { tags: { name: 'desc' } } });
     expect(mock.mock.calls.length).toBe(1);
     expect(mock.mock.calls[0][0]).toMatch('select "b0"."uuid_pk", "b0"."created_at", "b0"."title", "b0"."price", "b0".price * 1.19 as "price_taxed", "b0"."double", "b0"."meta", "b0"."author_id", "b0"."publisher_id", ' +
-      '"t1"."id" as "t1__id", "t1"."name" as "t1__name", "b0".price * 1.19 as "price_taxed" ' +
+      '"t1"."id" as "t1__id", "t1"."name" as "t1__name" ' +
       'from "book2" as "b0" ' +
       'left join "book2_tags" as "b2" on "b0"."uuid_pk" = "b2"."book2_uuid_pk" ' +
       'left join "book_tag2" as "t1" on "b2"."book_tag2_id" = "t1"."id" ' +
@@ -475,7 +475,7 @@ describe('Joined loading strategy', () => {
     expect(res1[0].test).toBeUndefined();
     expect(mock.mock.calls.length).toBe(1);
     expect(mock.mock.calls[0][0]).toMatch('select "b0"."uuid_pk", "b0"."created_at", "b0"."title", "b0"."perex", "b0"."price", "b0".price * 1.19 as "price_taxed", "b0"."double", "b0"."meta", "b0"."author_id", "b0"."publisher_id", ' +
-      '"a1"."id" as "a1__id", "a1"."created_at" as "a1__created_at", "a1"."updated_at" as "a1__updated_at", "a1"."name" as "a1__name", "a1"."email" as "a1__email", "a1"."age" as "a1__age", "a1"."terms_accepted" as "a1__terms_accepted", "a1"."optional" as "a1__optional", "a1"."identities" as "a1__identities", "a1"."born" as "a1__born", "a1"."born_time" as "a1__born_time", "a1"."favourite_book_uuid_pk" as "a1__favourite_book_uuid_pk", "a1"."favourite_author_id" as "a1__favourite_author_id", "b0".price * 1.19 as "price_taxed" ' +
+      '"a1"."id" as "a1__id", "a1"."created_at" as "a1__created_at", "a1"."updated_at" as "a1__updated_at", "a1"."name" as "a1__name", "a1"."email" as "a1__email", "a1"."age" as "a1__age", "a1"."terms_accepted" as "a1__terms_accepted", "a1"."optional" as "a1__optional", "a1"."identities" as "a1__identities", "a1"."born" as "a1__born", "a1"."born_time" as "a1__born_time", "a1"."favourite_book_uuid_pk" as "a1__favourite_book_uuid_pk", "a1"."favourite_author_id" as "a1__favourite_author_id" ' +
       'from "book2" as "b0" ' +
       'left join "author2" as "a1" on "b0"."author_id" = "a1"."id" ' +
       'where "b0"."author_id" is not null and "a1"."name" = $1');
@@ -488,7 +488,7 @@ describe('Joined loading strategy', () => {
     expect(mock.mock.calls[0][0]).toMatch('select "b0"."uuid_pk", "b0"."created_at", "b0"."title", "b0"."perex", "b0"."price", "b0".price * 1.19 as "price_taxed", "b0"."double", "b0"."meta", "b0"."author_id", "b0"."publisher_id", ' +
       '"a1"."id" as "a1__id", "a1"."created_at" as "a1__created_at", "a1"."updated_at" as "a1__updated_at", "a1"."name" as "a1__name", "a1"."email" as "a1__email", "a1"."age" as "a1__age", "a1"."terms_accepted" as "a1__terms_accepted", "a1"."optional" as "a1__optional", "a1"."identities" as "a1__identities", "a1"."born" as "a1__born", "a1"."born_time" as "a1__born_time", "a1"."favourite_book_uuid_pk" as "a1__favourite_book_uuid_pk", "a1"."favourite_author_id" as "a1__favourite_author_id", ' +
       '"f2"."uuid_pk" as "f2__uuid_pk", "f2"."created_at" as "f2__created_at", "f2"."title" as "f2__title", "f2"."price" as "f2__price", "f2".price * 1.19 as "f2__price_taxed", "f2"."double" as "f2__double", "f2"."meta" as "f2__meta", "f2"."author_id" as "f2__author_id", "f2"."publisher_id" as "f2__publisher_id", ' +
-      '"a3"."id" as "a3__id", "a3"."created_at" as "a3__created_at", "a3"."updated_at" as "a3__updated_at", "a3"."name" as "a3__name", "a3"."email" as "a3__email", "a3"."age" as "a3__age", "a3"."terms_accepted" as "a3__terms_accepted", "a3"."optional" as "a3__optional", "a3"."identities" as "a3__identities", "a3"."born" as "a3__born", "a3"."born_time" as "a3__born_time", "a3"."favourite_book_uuid_pk" as "a3__favourite_book_uuid_pk", "a3"."favourite_author_id" as "a3__favourite_author_id", "b0".price * 1.19 as "price_taxed" ' +
+      '"a3"."id" as "a3__id", "a3"."created_at" as "a3__created_at", "a3"."updated_at" as "a3__updated_at", "a3"."name" as "a3__name", "a3"."email" as "a3__email", "a3"."age" as "a3__age", "a3"."terms_accepted" as "a3__terms_accepted", "a3"."optional" as "a3__optional", "a3"."identities" as "a3__identities", "a3"."born" as "a3__born", "a3"."born_time" as "a3__born_time", "a3"."favourite_book_uuid_pk" as "a3__favourite_book_uuid_pk", "a3"."favourite_author_id" as "a3__favourite_author_id" ' +
       'from "book2" as "b0" ' +
       'left join "author2" as "a1" on "b0"."author_id" = "a1"."id" ' +
       'left join "book2" as "f2" on "a1"."favourite_book_uuid_pk" = "f2"."uuid_pk" ' +
@@ -513,8 +513,7 @@ describe('Joined loading strategy', () => {
     expect(mock.mock.calls[0][0]).toMatch('select "b0"."uuid_pk", "b0"."created_at", "b0"."title", "b0"."perex", "b0"."price", "b0".price * 1.19 as "price_taxed", "b0"."double", "b0"."meta", "b0"."author_id", "b0"."publisher_id", ' +
       '"a1"."id" as "a1__id", "a1"."created_at" as "a1__created_at", "a1"."updated_at" as "a1__updated_at", "a1"."name" as "a1__name", "a1"."email" as "a1__email", "a1"."age" as "a1__age", "a1"."terms_accepted" as "a1__terms_accepted", "a1"."optional" as "a1__optional", "a1"."identities" as "a1__identities", "a1"."born" as "a1__born", "a1"."born_time" as "a1__born_time", "a1"."favourite_book_uuid_pk" as "a1__favourite_book_uuid_pk", "a1"."favourite_author_id" as "a1__favourite_author_id", ' +
       '"f2"."uuid_pk" as "f2__uuid_pk", "f2"."created_at" as "f2__created_at", "f2"."title" as "f2__title", "f2"."price" as "f2__price", "f2".price * 1.19 as "f2__price_taxed", "f2"."double" as "f2__double", "f2"."meta" as "f2__meta", "f2"."author_id" as "f2__author_id", "f2"."publisher_id" as "f2__publisher_id", ' +
-      '"a3"."id" as "a3__id", "a3"."created_at" as "a3__created_at", "a3"."updated_at" as "a3__updated_at", "a3"."name" as "a3__name", "a3"."email" as "a3__email", "a3"."age" as "a3__age", "a3"."terms_accepted" as "a3__terms_accepted", "a3"."optional" as "a3__optional", "a3"."identities" as "a3__identities", "a3"."born" as "a3__born", "a3"."born_time" as "a3__born_time", "a3"."favourite_book_uuid_pk" as "a3__favourite_book_uuid_pk", "a3"."favourite_author_id" as "a3__favourite_author_id", ' +
-      '"b0".price * 1.19 as "price_taxed" ' +
+      '"a3"."id" as "a3__id", "a3"."created_at" as "a3__created_at", "a3"."updated_at" as "a3__updated_at", "a3"."name" as "a3__name", "a3"."email" as "a3__email", "a3"."age" as "a3__age", "a3"."terms_accepted" as "a3__terms_accepted", "a3"."optional" as "a3__optional", "a3"."identities" as "a3__identities", "a3"."born" as "a3__born", "a3"."born_time" as "a3__born_time", "a3"."favourite_book_uuid_pk" as "a3__favourite_book_uuid_pk", "a3"."favourite_author_id" as "a3__favourite_author_id" ' +
       'from "book2" as "b0" left join "author2" as "a1" on "b0"."author_id" = "a1"."id" ' +
       'left join "book2" as "f2" on "a1"."favourite_book_uuid_pk" = "f2"."uuid_pk" ' +
       'left join "author2" as "a3" on "f2"."author_id" = "a3"."id" ' +

tests/features/optimistic-lock/GH3440.test.ts

@@ -90,7 +90,7 @@ test(`GH issue 3440`, async () => {
   expect(mock.mock.calls).toHaveLength(9);
   expect(mock.mock.calls[0][0]).toMatch('begin');
   expect(mock.mock.calls[1][0]).toMatch("insert into `couch` (`id`, `user_id`, `name`) values (X'aaaaaaaac65f42b8408a034a6948448f', X'bbbbbbbbc65f42b8408a034a6948448f', 'n1')");
-  expect(mock.mock.calls[2][0]).toMatch('select `c0`.`id`, `c0`.`version`, `c0`.`version` from `couch` as `c0` where `c0`.`id` in (X\'aaaaaaaac65f42b8408a034a6948448f\')');
+  expect(mock.mock.calls[2][0]).toMatch('select `c0`.`id`, `c0`.`version` from `couch` as `c0` where `c0`.`id` in (X\'aaaaaaaac65f42b8408a034a6948448f\')');
   expect(mock.mock.calls[3][0]).toMatch('commit');
   expect(mock.mock.calls[4][0]).toMatch("select `c0`.* from `couch` as `c0` where `c0`.`id` = X'aaaaaaaac65f42b8408a034a6948448f' limit 1");
   expect(mock.mock.calls[5][0]).toMatch('begin');