refactor: move all BLOB and Buffer to ArrayBuffer (breaking change) (#…

…476)

* refactor: move all BLOB and Buffer to ArrayBuffer (breaking change)

* add exponential growth to the PipeArrayBuffer

* allow to preemptively allocate space and do that in leb encoding

Co-authored-by: Kyle Peacock <kylpeacock@gmail.com>

apps/sw-cert/package.json

@@ -6,7 +6,6 @@
     "@types/pako": "^1.0.1",
     "assert": "^2.0.0",
     "base64-arraybuffer": "^0.2.0",
-    "buffer": "^6.0.3",
     "events": "^3.2.0",
     "html-webpack-plugin": "^5.1.0",
     "http-server": "^0.12.3",

apps/sw-cert/src/sw/validation.ts

@@ -3,10 +3,9 @@ import {
   Certificate,
   HashTree,
   HttpAgent,
-  lookupPathEx,
+  lookup_path,
   reconstruct,
 } from '@dfinity/agent';
-import { blobFromUint8Array, blobToUint8Array } from '@dfinity/candid';
 import { Principal } from '@dfinity/principal';
 
 /**
@@ -29,10 +28,7 @@ export async function validateBody(
   agent: HttpAgent,
   shouldFetchRootKey = false,
 ): Promise<boolean> {
-  const cert = new Certificate(
-    { certificate: blobFromUint8Array(new Uint8Array(certificate)) },
-    agent,
-  );
+  const cert = new Certificate({ certificate: new Uint8Array(certificate) }, agent);
 
   // If we're running locally, update the key manually.
   if (shouldFetchRootKey) {
@@ -46,7 +42,7 @@ export async function validateBody(
 
   const hashTree: HashTree = cbor.decode(new Uint8Array(tree));
   const reconstructed = await reconstruct(hashTree);
-  const witness = cert.lookupEx(['canister', canisterId.toUint8Array(), 'certified_data']);
+  const witness = cert.lookup(['canister', canisterId.toUint8Array(), 'certified_data']);
 
   if (!witness) {
     throw new Error('Could not find certified data for this canister in the certificate.');
@@ -60,11 +56,11 @@ export async function validateBody(
 
   // Next, calculate the SHA of the content.
   const sha = await crypto.subtle.digest('SHA-256', body);
-  let treeSha = lookupPathEx(['http_assets', path], hashTree);
+  let treeSha = lookup_path(['http_assets', path], hashTree);
 
   if (!treeSha) {
     // Allow fallback to `index.html`.
-    treeSha = lookupPathEx(['http_assets', '/index.html'], hashTree);
+    treeSha = lookup_path(['http_assets', '/index.html'], hashTree);
   }
 
   if (!treeSha) {

apps/sw-cert/webpack.config.js

@@ -57,7 +57,6 @@ module.exports = {
     extensions: ['.tsx', '.ts', '.js'],
     fallback: {
       assert: require.resolve('assert/'),
-      buffer: require.resolve('buffer/'),
       events: require.resolve('events/'),
       stream: require.resolve('stream-browserify/'),
       util: require.resolve('util/'),
@@ -70,7 +69,6 @@ module.exports = {
       chunks: ['bundle'],
     }),
     new webpack.ProvidePlugin({
-      Buffer: [require.resolve('buffer/'), 'Buffer'],
       process: require.resolve('process/browser'),
     }),
     new webpack.EnvironmentPlugin({

demos/sample-javascript/package.json

@@ -7,7 +7,6 @@
     "@dfinity/identity": "^0.9.3",
     "@dfinity/principal": "^0.9.3",
     "assert": "^2.0.0",
-    "buffer": "^6.0.3",
     "events": "^3.2.0",
     "html-webpack-plugin": "^5.1.0",
     "process": "^0.11.10",

demos/sample-javascript/webpack.config.js

@@ -11,6 +11,9 @@ module.exports = {
   output: {
     path: path.join(__dirname, 'dist'),
   },
+  performance: {
+    hints: false,
+  },
   optimization: {
     splitChunks: {
       chunks: 'all',
@@ -40,7 +43,6 @@ module.exports = {
     },
     fallback: {
       assert: require.resolve('assert/'),
-      buffer: require.resolve('buffer/'),
       events: require.resolve('events/'),
       stream: require.resolve('stream-browserify/'),
       util: require.resolve('util/'),
@@ -53,7 +55,6 @@ module.exports = {
       filename: 'index.html',
     }),
     new webpack.ProvidePlugin({
-      Buffer: [require.resolve('buffer/'), 'Buffer'],
       process: require.resolve('process/browser'),
     }),
   ],

e2e/node/basic/basic.test.ts

@@ -1,26 +1,31 @@
+/**
+ * @jest-environment node
+ */
 import { Certificate, getManagementCanister } from '@dfinity/agent';
-import { blobFromText, IDL } from '@dfinity/candid';
+import { IDL } from '@dfinity/candid';
 import { Principal } from '@dfinity/principal';
 import agent from '../utils/agent';
-import { Buffer } from 'buffer/';
 
 test('read_state', async () => {
   const resolvedAgent = await agent;
   const now = Date.now() / 1000;
-  const path = [blobFromText('time')];
+  const path = [new TextEncoder().encode('time')];
   const response = await resolvedAgent.readState(Principal.fromHex('00000000000000000001'), {
     paths: [path],
   });
   const cert = new Certificate(response, resolvedAgent);
 
   expect(() => cert.lookup(path)).toThrow(/Cannot lookup unverified certificate/);
   expect(await cert.verify()).toBe(true);
-  expect(cert.lookup([blobFromText('Time')])).toBe(undefined);
+  expect(cert.lookup([new TextEncoder().encode('Time')])).toBe(undefined);
   // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
   const rawTime = cert.lookup(path)!;
   const decoded = IDL.decode(
     [IDL.Nat],
-    Buffer.concat([Buffer.from('DIDL\x00\x01\x7d'), rawTime]),
+    new Uint8Array([
+      ...new TextEncoder().encode('DIDL\x00\x01\x7d'),
+      ...(new Uint8Array(rawTime) || []),
+    ]),
   )[0];
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   const time = Number(decoded as any) / 1e9;

e2e/node/basic/counter.test.ts

@@ -1,3 +1,6 @@
+/**
+ * @jest-environment node
+ */
 import counterCanister from '../canisters/counter';
 
 test('counter', async () => {

e2e/node/basic/identity.test.ts

@@ -1,3 +1,6 @@
+/**
+ * @jest-environment node
+ */
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import { Actor, HttpAgent, SignIdentity } from '@dfinity/agent';
 import { IDL } from '@dfinity/candid';

e2e/node/basic/mitm.test.ts

@@ -1,3 +1,6 @@
+/**
+ * @jest-environment node
+ */
 import counterCanister from '../canisters/counter';
 
 let mitmTest = test;

e2e/node/canisters/counter.ts

@@ -1,5 +1,5 @@
 import { Actor } from '@dfinity/agent';
-import { blobFromUint8Array, IDL } from '@dfinity/candid';
+import { IDL } from '@dfinity/candid';
 import { Principal } from '@dfinity/principal';
 import { readFileSync } from 'fs';
 import path from 'path';
@@ -20,10 +20,10 @@ export default async function (): Promise<{
   actor: any;
 }> {
   if (!cache) {
-    const wasm = readFileSync(path.join(__dirname, 'counter.wasm'));
+    const module = readFileSync(path.join(__dirname, 'counter.wasm'));
 
     const canisterId = await Actor.createCanister({ agent: await agent });
-    await Actor.install({ module: blobFromUint8Array(wasm) }, { canisterId, agent: await agent });
+    await Actor.install({ module }, { canisterId, agent: await agent });
     const idl: IDL.InterfaceFactory = ({ IDL }) => {
       return IDL.Service({
         inc: IDL.Func([], [], []),

e2e/node/canisters/identity.ts

@@ -1,5 +1,5 @@
 import { Actor } from '@dfinity/agent';
-import { blobFromUint8Array, IDL } from '@dfinity/candid';
+import { IDL } from '@dfinity/candid';
 import { Principal } from '@dfinity/principal';
 import { readFileSync } from 'fs';
 import path from 'path';
@@ -22,10 +22,10 @@ export default async function (): Promise<{
   idl: IDL.InterfaceFactory;
 }> {
   if (!cache) {
-    const wasm = readFileSync(path.join(__dirname, 'identity.wasm'));
+    const module = readFileSync(path.join(__dirname, 'identity.wasm'));
 
     const canisterId = await Actor.createCanister({ agent: await agent });
-    await Actor.install({ module: blobFromUint8Array(wasm) }, { canisterId, agent: await agent });
+    await Actor.install({ module }, { canisterId, agent: await agent });
     const idl: IDL.InterfaceFactory = ({ IDL }) => {
       return IDL.Service({
         whoami: IDL.Func([], [IDL.Principal], []),

e2e/node/package.json

@@ -26,7 +26,6 @@
     "@types/node": "^13.7.7",
     "@typescript-eslint/eslint-plugin": "^4.14.2",
     "@typescript-eslint/parser": "^4.14.2",
-    "buffer": "^6.0.3",
     "eslint": "^7.19.0",
     "eslint-plugin-jsdoc": "^31.6.0",
     "jest": "^26.6.3",

e2e/node/test-setup.ts

@@ -9,4 +9,5 @@
 
 global.crypto = require('@trust/webcrypto');
 global.TextEncoder = require('text-encoding').TextEncoder; // eslint-disable-line
+global.TextDecoder = require('text-encoding').TextDecoder; // eslint-disable-line
 global.fetch = require('node-fetch');