From 633dc60bd85d5dd9035140c8b5ea9a2299e8574d Mon Sep 17 00:00:00 2001 From: Frederik Rothenberger Date: Thu, 7 Apr 2022 10:14:27 +0200 Subject: [PATCH 01/10] Add IDL.Unknown for deserializing unknown types --- packages/candid/src/idl.test.ts | 59 +++++++++++++++++++++++++++++ packages/candid/src/idl.ts | 66 +++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+) diff --git a/packages/candid/src/idl.test.ts b/packages/candid/src/idl.test.ts index 43813da2d..85b94c887 100644 --- a/packages/candid/src/idl.test.ts +++ b/packages/candid/src/idl.test.ts @@ -5,6 +5,7 @@ import * as IDL from './idl'; import { Principal } from '@dfinity/principal'; import { fromHexString, toHexString } from './utils/buffer'; +import { idlLabelToId } from './utils/hash'; function testEncode(typ: IDL.Type, val: any, hex: string, _str: string) { expect(toHexString(IDL.encode([typ], [val]))).toEqual(hex); @@ -416,3 +417,61 @@ test('IDL encoding (multiple arguments)', () => { test('Stringify bigint', () => { expect(() => IDL.encode([IDL.Nat], [{ x: BigInt(42) }])).toThrow(/Invalid nat argument/); }); + +function hashedPropertyName(name: string) { + return '_' + idlLabelToId(name) + '_'; +} + +test('IDL unknown decoding', () => { + const decodedType = IDL.Variant({ _24860_: IDL.Text, _5048165_: IDL.Text }); + const encoded = '4449444c016b029cc20171e58eb4027101000004676f6f64'; + + const value = IDL.decode([IDL.Unknown], fromHexString(encoded))[0] as any; + expect(value[hashedPropertyName('ok')]).toEqual('good'); + expect(value.type()).toEqual(decodedType); + + const reencoded = toHexString(IDL.encode([value.type()], [value])); + expect(reencoded).toEqual(encoded); +}); + +test('IDL unknown nested decoding', () => { + const nestedType = IDL.Record({ foo: IDL.Int32, bar: IDL.Bool }); + const recordType = IDL.Record({ + foo: IDL.Int32, + bar: nestedType, + baz: nestedType, + bib: nestedType, + }); + + const recordUnknownType = IDL.Record({ + foo: IDL.Int32, + bar: IDL.Unknown, + baz: nestedType, + bib: nestedType, + }); + + const nestedHashedType = IDL.Record({ _5097222_: IDL.Int32, _4895187_: IDL.Bool }); + const recordHashedType = IDL.Record({ + foo: IDL.Int32, + bar: nestedHashedType, + baz: nestedType, + bib: nestedType, + }); + + const encoded = + '4449444c026c02d3e3aa027e868eb702756c04d3e3aa0200dbe3aa0200bbf1aa0200868eb702750101012a000000012a000000012a0000002a000000'; + const nestedValue = { foo: 42, bar: true }; + const value = { foo: 42, bar: nestedValue, baz: nestedValue, bib: nestedValue }; + + const decodedValue = IDL.decode([recordUnknownType], fromHexString(encoded))[0] as any; + expect(decodedValue).toHaveProperty('bar'); + expect(decodedValue.bar[hashedPropertyName('foo')]).toEqual(42); + expect(decodedValue.bar[hashedPropertyName('bar')]).toEqual(true); + expect(decodedValue.baz).toEqual(value.baz); + expect(decodedValue.bar.type()).toEqual(nestedHashedType); + + const reencoded = toHexString(IDL.encode([recordHashedType], [decodedValue])); + // expect(reencoded).toEqual(encoded); does not hold because type table is different + const decodedValue2 = IDL.decode([recordType], fromHexString(reencoded))[0] as any; + expect(decodedValue2).toEqual(value); +}); diff --git a/packages/candid/src/idl.ts b/packages/candid/src/idl.ts index 004378f9f..08f4e48e8 100644 --- a/packages/candid/src/idl.ts +++ b/packages/candid/src/idl.ts @@ -280,6 +280,70 @@ export class EmptyClass extends PrimitiveType { } } +/** + * Represents an IDL Unknown, a placeholder type for deserialization only. + * When decoding a value as Unknown, all fields will be retained but the names are only available in + * hashed form. + * A deserialized unknown will offer it's actual type by calling the `type()` function. + * Unknown cannot be serialized and attempting to do so will throw an error. + */ +export class UnknownClass extends Type { + public checkType(t: Type): Type { + throw new Error('Method not implemented for unknown.'); + } + + public accept(v: Visitor, d: D): R { + throw v.visitType(this, d); + } + + public covariant(x: any): x is any { + return false; + } + + public encodeValue(): never { + throw new Error('Unknown cannot appear as a function argument'); + } + + public valueToString(): never { + throw new Error('Unknown cannot appear as a value'); + } + + public encodeType(): never { + throw new Error('Unknown cannot be serialized'); + } + + public decodeValue(b: Pipe, t: Type): any { + const decodedValue = t.decodeValue(b, t); + let typeFunc; + if (t instanceof RecClass) { + typeFunc = () => t.getType(); + } else { + typeFunc = () => t; + } + + // Do not use 'decodedValue.type = typeFunc' because this would lead to an enumerable property + // 'type' which means it would be serialized if the value would be candid encoded again. + // This in turn leads to problems if the decoded value is a variant because these values are + // only allowed to have a single property. + Object.defineProperty(decodedValue, 'type', { + value: typeFunc, + writable: true, + enumerable: false, + configurable: true, + }); + return decodedValue; + } + + protected _buildTypeTableImpl(): void { + throw new Error('Method not implemented for unknown.'); + } + + get name() { + return 'Unknown'; + } +} + + /** * Represents an IDL Bool */ @@ -1562,6 +1626,7 @@ export type InterfaceFactory = (idl: { IDL: { Empty: EmptyClass; Reserved: ReservedClass; + Unknown: UnknownClass; Bool: BoolClass; Null: NullClass; Text: TextClass; @@ -1598,6 +1663,7 @@ export type InterfaceFactory = (idl: { // Export Types instances. export const Empty = new EmptyClass(); export const Reserved = new ReservedClass(); +export const Unknown = new UnknownClass(); export const Bool = new BoolClass(); export const Null = new NullClass(); export const Text = new TextClass(); From 9daf26e516753291b5e49ac7946bd343c269f1d4 Mon Sep 17 00:00:00 2001 From: Frederik Rothenberger Date: Thu, 7 Apr 2022 10:15:37 +0200 Subject: [PATCH 02/10] Reformatting with prettier --- packages/candid/src/idl.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/candid/src/idl.ts b/packages/candid/src/idl.ts index 08f4e48e8..31262a383 100644 --- a/packages/candid/src/idl.ts +++ b/packages/candid/src/idl.ts @@ -343,7 +343,6 @@ export class UnknownClass extends Type { } } - /** * Represents an IDL Bool */ From f9bd596d136c0852c1b64af40e54ac93f5036131 Mon Sep 17 00:00:00 2001 From: Frederik Rothenberger Date: Thu, 7 Apr 2022 14:26:12 +0200 Subject: [PATCH 03/10] More tests --- packages/candid/src/idl.test.ts | 114 +++++++++++++++++++++++++++++++- packages/candid/src/idl.ts | 11 ++- 2 files changed, 121 insertions(+), 4 deletions(-) diff --git a/packages/candid/src/idl.test.ts b/packages/candid/src/idl.test.ts index 85b94c887..07eb913b6 100644 --- a/packages/candid/src/idl.test.ts +++ b/packages/candid/src/idl.test.ts @@ -422,7 +422,7 @@ function hashedPropertyName(name: string) { return '_' + idlLabelToId(name) + '_'; } -test('IDL unknown decoding', () => { +test('decode / encode unknown variant', () => { const decodedType = IDL.Variant({ _24860_: IDL.Text, _5048165_: IDL.Text }); const encoded = '4449444c016b029cc20171e58eb4027101000004676f6f64'; @@ -434,7 +434,117 @@ test('IDL unknown decoding', () => { expect(reencoded).toEqual(encoded); }); -test('IDL unknown nested decoding', () => { +test('decode unknown text', () => { + const text = IDL.decode([IDL.Unknown], fromHexString('4449444c00017107486920e298830a'))[0] as any; + expect(text.valueOf()).toEqual('Hi ☃\n'); + expect(text.type().name).toEqual(IDL.Text.name); +}); + +test('decode unknown int', () => { + const int = IDL.decode([IDL.Unknown], fromHexString('4449444c00017c2a'))[0] as any; + expect(int.valueOf()).toEqual(BigInt(42)); + expect(int.type().name).toEqual(IDL.Int.name); +}); + +test('decode unknown nat', () => { + const nat = IDL.decode([IDL.Unknown], fromHexString('4449444c00017d2a'))[0] as any; + expect(nat.valueOf()).toEqual(BigInt(42)); + expect(nat.type().name).toEqual(IDL.Nat.name); +}); + +test('decode unknown null', () => { + const value = IDL.decode([IDL.Unknown], fromHexString('4449444c00017f'))[0] as any; + // expect(value.valueOf()).toEqual(null); TODO: This does not hold. What do we do about this? + expect(value.type().name).toEqual(IDL.Null.name); +}); + +test('decode unknown bool', () => { + const value = IDL.decode([IDL.Unknown], fromHexString('4449444c00017e01'))[0] as any; + expect(value.valueOf()).toEqual(true); + expect(value.type().name).toEqual(IDL.Bool.name); +}); + +test('decode unknown fixed-width number', () => { + const int8 = IDL.decode([IDL.Unknown], fromHexString('4449444c0001777f'))[0] as any; + expect(int8.valueOf()).toEqual(127); + expect(int8.type().name).toEqual(IDL.Int8.name); + + const int32 = IDL.decode([IDL.Unknown], fromHexString('4449444c000175d2029649'))[0] as any; + expect(int32.valueOf()).toEqual(1234567890); + expect(int32.type().name).toEqual(IDL.Int32.name); + + const int64 = IDL.decode( + [IDL.Unknown], + fromHexString('4449444c0001742a00000000000000'), + )[0] as any; + expect(int64.valueOf()).toEqual(BigInt(42)); + expect(int64.type().name).toEqual(IDL.Int64.name); + + const nat8 = IDL.decode([IDL.Unknown], fromHexString('4449444c00017b2a'))[0] as any; + expect(nat8.valueOf()).toEqual(42); + expect(nat8.type().name).toEqual(IDL.Nat8.name); + + const nat32 = IDL.decode([IDL.Unknown], fromHexString('4449444c0001792a000000'))[0] as any; + expect(nat32.valueOf()).toEqual(42); + expect(nat32.type().name).toEqual(IDL.Nat32.name); + + const nat64 = IDL.decode( + [IDL.Unknown], + fromHexString('4449444c000178d202964900000000'), + )[0] as any; + expect(nat64.valueOf()).toEqual(BigInt(1234567890)); + expect(nat64.type().name).toEqual(IDL.Nat64.name); +}); + +test('decode unknown float', () => { + const float64 = IDL.decode( + [IDL.Unknown], + fromHexString('4449444c0001720000000000001840'), + )[0] as any; + expect(float64.valueOf()).toEqual(6); + expect(float64.type().name).toEqual(IDL.Float64.name); + + const nan = IDL.decode([IDL.Unknown], fromHexString('4449444c000172000000000000f87f'))[0] as any; + expect(nan.valueOf()).toEqual(Number.NaN); + expect(nan.type().name).toEqual(IDL.Float64.name); + + const infinity = IDL.decode( + [IDL.Unknown], + fromHexString('4449444c000172000000000000f07f'), + )[0] as any; + expect(infinity.valueOf()).toEqual(Number.POSITIVE_INFINITY); + expect(infinity.type().name).toEqual(IDL.Float64.name); +}); + +test('decode unknown vec of tuples', () => { + const encoded = '4449444c026c02007c01716d000101012a0474657874'; + const value = IDL.decode([IDL.Unknown], fromHexString(encoded))[0] as any; + expect(value).toEqual([[BigInt(42), 'text']]); + const reencoded = toHexString(IDL.encode([value.type()], [value])); + expect(reencoded).toEqual(encoded); +}); + +test('decode / encode unknown mutual recursive lists', () => { + // original types + const List1 = IDL.Rec(); + const List2 = IDL.Rec(); + List1.fill(IDL.Opt(List2)); + List2.fill(IDL.Record({ head: IDL.Int, tail: List1 })); + + const encoded = '4449444c026e016c02a0d2aca8047c90eddae7040001000101010200'; + const value = IDL.decode([IDL.Unknown], fromHexString(encoded))[0] as any; + expect(value).toEqual([ + { _1158359328_: BigInt(1), _1291237008_: [{ _1158359328_: BigInt(2), _1291237008_: [] }] }, + ]); + + const reencoded = toHexString(IDL.encode([value.type()], [value])); + // expect(reencoded).toEqual(encoded); does not hold because type table gets built differently + // however the result is still compatible with original types: + const value2 = IDL.decode([List1], fromHexString(reencoded))[0]; + expect(value2).toEqual([{ head: BigInt(1), tail: [{ head: BigInt(2), tail: [] }] }]); +}); + +test('decode / encode unknown nested record', () => { const nestedType = IDL.Record({ foo: IDL.Int32, bar: IDL.Bool }); const recordType = IDL.Record({ foo: IDL.Int32, diff --git a/packages/candid/src/idl.ts b/packages/candid/src/idl.ts index 31262a383..0e6c44695 100644 --- a/packages/candid/src/idl.ts +++ b/packages/candid/src/idl.ts @@ -313,14 +313,21 @@ export class UnknownClass extends Type { } public decodeValue(b: Pipe, t: Type): any { - const decodedValue = t.decodeValue(b, t); + let decodedValue = t.decodeValue(b, t); + + if (Object(decodedValue) !== decodedValue) { + // decodedValue is primitive. Box it, otherwise we cannot add the type() function. + // The type() function is important for primitives because otherwise we cannot tell apart the + // different number types. + decodedValue = Object(decodedValue); + } + let typeFunc; if (t instanceof RecClass) { typeFunc = () => t.getType(); } else { typeFunc = () => t; } - // Do not use 'decodedValue.type = typeFunc' because this would lead to an enumerable property // 'type' which means it would be serialized if the value would be candid encoded again. // This in turn leads to problems if the decoded value is a variant because these values are From 8f27242a13d9560ff3a26f0bc460f3b32e3fe906 Mon Sep 17 00:00:00 2001 From: Frederik Rothenberger Date: Thu, 7 Apr 2022 14:30:39 +0200 Subject: [PATCH 04/10] Add changelog entry --- docs/generated/changelog.html | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/generated/changelog.html b/docs/generated/changelog.html index 2165722f8..8f75ff1c9 100644 --- a/docs/generated/changelog.html +++ b/docs/generated/changelog.html @@ -27,6 +27,7 @@

Version 0.10.5

Versioning tool now sets patch version to 0 for minor version updates, or patch and minor versions to 0 for major version updates +
  • Candid now allows decoding unknown values using IDL.Unknown as a type placeholder.

    • Version 0.10.3

      From 92531d222b1c6ff5b20619c4104b75f880c258be Mon Sep 17 00:00:00 2001 From: Frederik Rothenberger Date: Thu, 7 Apr 2022 14:40:58 +0200 Subject: [PATCH 05/10] Add test for serializing unknown --- packages/candid/src/idl.test.ts | 7 ++++++- packages/candid/src/idl.ts | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/candid/src/idl.test.ts b/packages/candid/src/idl.test.ts index 07eb913b6..264318595 100644 --- a/packages/candid/src/idl.test.ts +++ b/packages/candid/src/idl.test.ts @@ -434,6 +434,10 @@ test('decode / encode unknown variant', () => { expect(reencoded).toEqual(encoded); }); +test('throw on serializing unknown', () => { + expect(() => IDL.encode([IDL.Unknown], ['test'])).toThrow('Unknown cannot be serialized'); +}); + test('decode unknown text', () => { const text = IDL.decode([IDL.Unknown], fromHexString('4449444c00017107486920e298830a'))[0] as any; expect(text.valueOf()).toEqual('Hi ☃\n'); @@ -538,7 +542,7 @@ test('decode / encode unknown mutual recursive lists', () => { ]); const reencoded = toHexString(IDL.encode([value.type()], [value])); - // expect(reencoded).toEqual(encoded); does not hold because type table gets built differently + // expect(reencoded).toEqual(encoded); does not hold because type table is different // however the result is still compatible with original types: const value2 = IDL.decode([List1], fromHexString(reencoded))[0]; expect(value2).toEqual([{ head: BigInt(1), tail: [{ head: BigInt(2), tail: [] }] }]); @@ -582,6 +586,7 @@ test('decode / encode unknown nested record', () => { const reencoded = toHexString(IDL.encode([recordHashedType], [decodedValue])); // expect(reencoded).toEqual(encoded); does not hold because type table is different + // however the result is still compatible with original types: const decodedValue2 = IDL.decode([recordType], fromHexString(reencoded))[0] as any; expect(decodedValue2).toEqual(value); }); diff --git a/packages/candid/src/idl.ts b/packages/candid/src/idl.ts index 0e6c44695..12171e6da 100644 --- a/packages/candid/src/idl.ts +++ b/packages/candid/src/idl.ts @@ -342,7 +342,7 @@ export class UnknownClass extends Type { } protected _buildTypeTableImpl(): void { - throw new Error('Method not implemented for unknown.'); + throw new Error('Unknown cannot be serialized'); } get name() { From 2a1705dc364b52168078a476765b8bd0cf579751 Mon Sep 17 00:00:00 2001 From: Frederik Rothenberger Date: Thu, 7 Apr 2022 14:41:53 +0200 Subject: [PATCH 06/10] Move helper to top --- packages/candid/src/idl.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/candid/src/idl.test.ts b/packages/candid/src/idl.test.ts index 264318595..2dec968d4 100644 --- a/packages/candid/src/idl.test.ts +++ b/packages/candid/src/idl.test.ts @@ -25,6 +25,10 @@ function test_args(typs: IDL.Type[], vals: any[], hex: string, _str: string) { expect(IDL.decode(typs, fromHexString(hex))).toEqual(vals); } +function hashedPropertyName(name: string) { + return '_' + idlLabelToId(name) + '_'; +} + test('IDL encoding (magic number)', () => { // Wrong magic number expect(() => IDL.decode([IDL.Nat], fromHexString('2a'))).toThrow( @@ -418,10 +422,6 @@ test('Stringify bigint', () => { expect(() => IDL.encode([IDL.Nat], [{ x: BigInt(42) }])).toThrow(/Invalid nat argument/); }); -function hashedPropertyName(name: string) { - return '_' + idlLabelToId(name) + '_'; -} - test('decode / encode unknown variant', () => { const decodedType = IDL.Variant({ _24860_: IDL.Text, _5048165_: IDL.Text }); const encoded = '4449444c016b029cc20171e58eb4027101000004676f6f64'; From 570222bd5d5d9976d75eae707487ac10c86cf1d1 Mon Sep 17 00:00:00 2001 From: Frederik Rothenberger Date: Thu, 7 Apr 2022 14:56:50 +0200 Subject: [PATCH 07/10] Add test for service --- packages/candid/src/idl.test.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/candid/src/idl.test.ts b/packages/candid/src/idl.test.ts index 2dec968d4..c0c20963f 100644 --- a/packages/candid/src/idl.test.ts +++ b/packages/candid/src/idl.test.ts @@ -528,6 +528,15 @@ test('decode unknown vec of tuples', () => { expect(reencoded).toEqual(encoded); }); +test('decode unknown service', () => { + const value = IDL.decode( + [IDL.Unknown], + fromHexString('4449444c026a0171017d00690103666f6f0001010103caffee'), + )[0] as any; + expect(value).toEqual(Principal.fromText('w7x7r-cok77-xa')); + expect(value.type()).toEqual(IDL.Service({})); +}); + test('decode / encode unknown mutual recursive lists', () => { // original types const List1 = IDL.Rec(); From 7de395fa6a33d3d7eb753ac0948ae8ad7bcee0fc Mon Sep 17 00:00:00 2001 From: Frederik Rothenberger Date: Thu, 7 Apr 2022 15:02:32 +0200 Subject: [PATCH 08/10] Add unit test for func --- packages/candid/src/idl.test.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/candid/src/idl.test.ts b/packages/candid/src/idl.test.ts index c0c20963f..afd60210e 100644 --- a/packages/candid/src/idl.test.ts +++ b/packages/candid/src/idl.test.ts @@ -537,6 +537,15 @@ test('decode unknown service', () => { expect(value.type()).toEqual(IDL.Service({})); }); +test('decode unknown func', () => { + const value = IDL.decode( + [IDL.Unknown], + fromHexString('4449444c016a0171017d01010100010103caffee03666f6f'), + )[0] as any; + expect(value).toEqual([Principal.fromText('w7x7r-cok77-xa'), 'foo']); + expect(value.type()).toEqual(IDL.Func([], [], [])); +}); + test('decode / encode unknown mutual recursive lists', () => { // original types const List1 = IDL.Rec(); From 64ee3a678ef9757f28c03e1f8d843ebe56e6ced1 Mon Sep 17 00:00:00 2001 From: Kyle Peacock Date: Thu, 7 Apr 2022 10:32:14 -0700 Subject: [PATCH 09/10] Update changelog.html --- docs/generated/changelog.html | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/generated/changelog.html b/docs/generated/changelog.html index 46e9dc2b7..5d75f50f0 100644 --- a/docs/generated/changelog.html +++ b/docs/generated/changelog.html @@ -28,10 +28,6 @@

      Version 0.10.5

      versions to 0 for major version updates
    • Removes jest-expect-message, which was making test error messages less useful
      • -
    • - Candid now allows decoding values with unknown types using 'IDL.Unknown' as a type - placeholder. -

    Version 0.10.3

      From 11f50f09ead256ae7c83292c56dbc8a5cef2ec4c Mon Sep 17 00:00:00 2001 From: Kyle Peacock Date: Thu, 7 Apr 2022 10:33:31 -0700 Subject: [PATCH 10/10] Update packages/candid/src/idl.ts --- packages/candid/src/idl.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/candid/src/idl.ts b/packages/candid/src/idl.ts index 12171e6da..d0a49e660 100644 --- a/packages/candid/src/idl.ts +++ b/packages/candid/src/idl.ts @@ -1669,6 +1669,9 @@ export type InterfaceFactory = (idl: { // Export Types instances. export const Empty = new EmptyClass(); export const Reserved = new ReservedClass(); +/** + * Client-only type for deserializing unknown data. Not supported by Candid, and its use is discouraged. + */ export const Unknown = new UnknownClass(); export const Bool = new BoolClass(); export const Null = new NullClass();