From c5d191135757f66db366b058dc3c22e85f8acba3 Mon Sep 17 00:00:00 2001 From: Rob Herley Date: Mon, 4 Apr 2022 17:54:15 -0400 Subject: [PATCH 01/10] add digest header for uploads --- .../src/internal/upload-http-client.ts | 7 +++++- packages/artifact/src/internal/utils.ts | 24 ++++++++++++++++--- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/packages/artifact/src/internal/upload-http-client.ts b/packages/artifact/src/internal/upload-http-client.ts index 9892a42d79..a9a77f327e 100644 --- a/packages/artifact/src/internal/upload-http-client.ts +++ b/packages/artifact/src/internal/upload-http-client.ts @@ -9,6 +9,7 @@ import { UploadResults } from './contracts' import { + digestForStream, getArtifactUrl, getContentRange, getUploadHeaders, @@ -406,6 +407,9 @@ export class UploadHttpClient { isGzip: boolean, totalFileSize: number ): Promise { + // open a new stream and read it to compute the digest + const digest = await digestForStream(openStream()) + // prepare all the necessary headers before making any http call const headers = getUploadHeaders( 'application/octet-stream', @@ -413,7 +417,8 @@ export class UploadHttpClient { isGzip, totalFileSize, end - start + 1, - getContentRange(start, end, uploadFileSize) + getContentRange(start, end, uploadFileSize), + digest ) const uploadChunkRequest = async (): Promise => { diff --git a/packages/artifact/src/internal/utils.ts b/packages/artifact/src/internal/utils.ts index f2b99f335c..e23056f64e 100644 --- a/packages/artifact/src/internal/utils.ts +++ b/packages/artifact/src/internal/utils.ts @@ -1,9 +1,10 @@ -import {debug, info, warning} from '@actions/core' +import * as crypto from 'crypto' import {promises as fs} from 'fs' +import {IncomingHttpHeaders} from 'http' +import {debug, info, warning} from '@actions/core' import {HttpCodes, HttpClient} from '@actions/http-client' import {BearerCredentialHandler} from '@actions/http-client/auth' import {IHeaders, IHttpClientResponse} from '@actions/http-client/interfaces' -import {IncomingHttpHeaders} from 'http' import { getRuntimeToken, getRuntimeUrl, @@ -180,7 +181,8 @@ export function getUploadHeaders( isGzip?: boolean, uncompressedLength?: number, contentLength?: number, - contentRange?: string + contentRange?: string, + digest?: string ): IHeaders { const requestOptions: IHeaders = {} requestOptions['Accept'] = `application/json;api-version=${getApiVersion()}` @@ -202,6 +204,10 @@ export function getUploadHeaders( if (contentRange) { requestOptions['Content-Range'] = contentRange } + if (digest) { + // TODO(robherley): should we use 'Digest' directly? https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Digest + requestOptions['X-Digest'] = `sha-256=${digest}` + } return requestOptions } @@ -291,3 +297,15 @@ export function getProperRetention( export async function sleep(milliseconds: number): Promise { return new Promise(resolve => setTimeout(resolve, milliseconds)) } + +export async function digestForStream( + stream: NodeJS.ReadableStream +): Promise { + return new Promise((resolve, reject) => { + // TODO(robherley): switch to crc64 for production + const hasher = crypto.createHash('sha256') + stream.on('data', data => hasher.update(data)) + stream.on('end', () => resolve(hasher.digest('hex'))) + stream.on('error', reject) + }) +} From e3549a9c58adcbd7a65ae37f43235fdfb3eef4fc Mon Sep 17 00:00:00 2001 From: Rob Herley Date: Tue, 5 Apr 2022 09:31:49 -0400 Subject: [PATCH 02/10] artifact: tests for stream digest util --- packages/artifact/__tests__/util.test.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/packages/artifact/__tests__/util.test.ts b/packages/artifact/__tests__/util.test.ts index c353cdc60a..73e3c78c63 100644 --- a/packages/artifact/__tests__/util.test.ts +++ b/packages/artifact/__tests__/util.test.ts @@ -10,6 +10,7 @@ import { getInitialRetryIntervalInMilliseconds, getRetryMultiplier } from '../src/internal/config-variables' +import {Readable} from 'stream' jest.mock('../src/internal/config-variables') @@ -74,15 +75,18 @@ describe('Utils', () => { const size = 24 const uncompressedLength = 100 const range = 'bytes 0-199/200' + const digest = + '5e2bf57d3f40c4b6df69daf1936cb766f832374b4fc0259a7cbff06e2f70f269' const headers = utils.getUploadHeaders( contentType, true, true, uncompressedLength, size, - range + range, + digest ) - expect(Object.keys(headers).length).toEqual(8) + expect(Object.keys(headers).length).toEqual(9) expect(headers['Accept']).toEqual( `application/json;api-version=${utils.getApiVersion()}` ) @@ -93,6 +97,7 @@ describe('Utils', () => { expect(headers['x-tfs-filelength']).toEqual(uncompressedLength) expect(headers['Content-Length']).toEqual(size) expect(headers['Content-Range']).toEqual(range) + expect(headers['X-Digest']).toEqual(`sha-256=${digest}`) }) it('Test constructing upload headers with only required parameter', () => { @@ -219,4 +224,14 @@ describe('Utils', () => { const size2 = (await fs.promises.stat(emptyFile2)).size expect(size2).toEqual(0) }) + + it('Creates a digest from a readable stream', async () => { + const data = 'lorem ipsum' + const stream = Readable.from(data) + const digest = await utils.digestForStream(stream) + + expect(digest).toBe( + '5e2bf57d3f40c4b6df69daf1936cb766f832374b4fc0259a7cbff06e2f70f269' + ) + }) }) From 3d61fe800013bdbafeb0e881286aaa7e03f04f4c Mon Sep 17 00:00:00 2001 From: Rob Herley Date: Tue, 26 Apr 2022 17:13:40 +0000 Subject: [PATCH 03/10] add crc64 & tests --- packages/artifact/__tests__/crc64.test.ts | 22 ++ packages/artifact/src/internal/crc64.ts | 299 ++++++++++++++++++++++ 2 files changed, 321 insertions(+) create mode 100644 packages/artifact/__tests__/crc64.test.ts create mode 100644 packages/artifact/src/internal/crc64.ts diff --git a/packages/artifact/__tests__/crc64.test.ts b/packages/artifact/__tests__/crc64.test.ts new file mode 100644 index 0000000000..6f5a2e03a0 --- /dev/null +++ b/packages/artifact/__tests__/crc64.test.ts @@ -0,0 +1,22 @@ +import CRC64 from '../src/internal/crc64' + +const fixtures = { + data: + '🚀 👉😎👉 Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.\nUt enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.\nDuis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.\nExcepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.', + expected: 'D170FCD035F4D958' +} + +describe('@actions/artifact/src/internal/crc64', () => { + it('CRC64 from string', async () => { + const crc = new CRC64() + crc.update(fixtures.data) + expect(crc.digest()).toEqual(fixtures.expected) + }) + + it('CRC64 from buffer', async () => { + const crc = new CRC64() + const buf = Buffer.from(fixtures.data) + crc.update(buf) + expect(crc.digest()).toEqual(fixtures.expected) + }) +}) diff --git a/packages/artifact/src/internal/crc64.ts b/packages/artifact/src/internal/crc64.ts new file mode 100644 index 0000000000..97fcc3df26 --- /dev/null +++ b/packages/artifact/src/internal/crc64.ts @@ -0,0 +1,299 @@ +/** + * CRC64: cyclic redundancy check, 64-bits + * + * In order to validate that artifacts are not being corrupted over the wire, this redundancy check allows us to + * validate that there was no corruption during transmission. The implementation here is based on Go's hash/crc64 pkg, + * but without the slicing-by-8 optimization: https://cs.opensource.google/go/go/+/master:src/hash/crc64/crc64.go + * + * This implementation uses a pregenerated table based on 0x9A6C9329AC4BC9B5 as the polynomial, the same polynomial that + * is used for Azure Storage: https://github.com/Azure/azure-storage-net + */ + +class CRC64 { + private _crc: bigint + // when transpile target is >= ES2020 (after dropping node 12) these can be changed to bigint literals - ts(2737) + private _table = [ + BigInt('0x0000000000000000'), + BigInt('0x7F6EF0C830358979'), + BigInt('0xFEDDE190606B12F2'), + BigInt('0x81B31158505E9B8B'), + BigInt('0xC962E5739841B68F'), + BigInt('0xB60C15BBA8743FF6'), + BigInt('0x37BF04E3F82AA47D'), + BigInt('0x48D1F42BC81F2D04'), + BigInt('0xA61CECB46814FE75'), + BigInt('0xD9721C7C5821770C'), + BigInt('0x58C10D24087FEC87'), + BigInt('0x27AFFDEC384A65FE'), + BigInt('0x6F7E09C7F05548FA'), + BigInt('0x1010F90FC060C183'), + BigInt('0x91A3E857903E5A08'), + BigInt('0xEECD189FA00BD371'), + BigInt('0x78E0FF3B88BE6F81'), + BigInt('0x078E0FF3B88BE6F8'), + BigInt('0x863D1EABE8D57D73'), + BigInt('0xF953EE63D8E0F40A'), + BigInt('0xB1821A4810FFD90E'), + BigInt('0xCEECEA8020CA5077'), + BigInt('0x4F5FFBD87094CBFC'), + BigInt('0x30310B1040A14285'), + BigInt('0xDEFC138FE0AA91F4'), + BigInt('0xA192E347D09F188D'), + BigInt('0x2021F21F80C18306'), + BigInt('0x5F4F02D7B0F40A7F'), + BigInt('0x179EF6FC78EB277B'), + BigInt('0x68F0063448DEAE02'), + BigInt('0xE943176C18803589'), + BigInt('0x962DE7A428B5BCF0'), + BigInt('0xF1C1FE77117CDF02'), + BigInt('0x8EAF0EBF2149567B'), + BigInt('0x0F1C1FE77117CDF0'), + BigInt('0x7072EF2F41224489'), + BigInt('0x38A31B04893D698D'), + BigInt('0x47CDEBCCB908E0F4'), + BigInt('0xC67EFA94E9567B7F'), + BigInt('0xB9100A5CD963F206'), + BigInt('0x57DD12C379682177'), + BigInt('0x28B3E20B495DA80E'), + BigInt('0xA900F35319033385'), + BigInt('0xD66E039B2936BAFC'), + BigInt('0x9EBFF7B0E12997F8'), + BigInt('0xE1D10778D11C1E81'), + BigInt('0x606216208142850A'), + BigInt('0x1F0CE6E8B1770C73'), + BigInt('0x8921014C99C2B083'), + BigInt('0xF64FF184A9F739FA'), + BigInt('0x77FCE0DCF9A9A271'), + BigInt('0x08921014C99C2B08'), + BigInt('0x4043E43F0183060C'), + BigInt('0x3F2D14F731B68F75'), + BigInt('0xBE9E05AF61E814FE'), + BigInt('0xC1F0F56751DD9D87'), + BigInt('0x2F3DEDF8F1D64EF6'), + BigInt('0x50531D30C1E3C78F'), + BigInt('0xD1E00C6891BD5C04'), + BigInt('0xAE8EFCA0A188D57D'), + BigInt('0xE65F088B6997F879'), + BigInt('0x9931F84359A27100'), + BigInt('0x1882E91B09FCEA8B'), + BigInt('0x67EC19D339C963F2'), + BigInt('0xD75ADABD7A6E2D6F'), + BigInt('0xA8342A754A5BA416'), + BigInt('0x29873B2D1A053F9D'), + BigInt('0x56E9CBE52A30B6E4'), + BigInt('0x1E383FCEE22F9BE0'), + BigInt('0x6156CF06D21A1299'), + BigInt('0xE0E5DE5E82448912'), + BigInt('0x9F8B2E96B271006B'), + BigInt('0x71463609127AD31A'), + BigInt('0x0E28C6C1224F5A63'), + BigInt('0x8F9BD7997211C1E8'), + BigInt('0xF0F5275142244891'), + BigInt('0xB824D37A8A3B6595'), + BigInt('0xC74A23B2BA0EECEC'), + BigInt('0x46F932EAEA507767'), + BigInt('0x3997C222DA65FE1E'), + BigInt('0xAFBA2586F2D042EE'), + BigInt('0xD0D4D54EC2E5CB97'), + BigInt('0x5167C41692BB501C'), + BigInt('0x2E0934DEA28ED965'), + BigInt('0x66D8C0F56A91F461'), + BigInt('0x19B6303D5AA47D18'), + BigInt('0x980521650AFAE693'), + BigInt('0xE76BD1AD3ACF6FEA'), + BigInt('0x09A6C9329AC4BC9B'), + BigInt('0x76C839FAAAF135E2'), + BigInt('0xF77B28A2FAAFAE69'), + BigInt('0x8815D86ACA9A2710'), + BigInt('0xC0C42C4102850A14'), + BigInt('0xBFAADC8932B0836D'), + BigInt('0x3E19CDD162EE18E6'), + BigInt('0x41773D1952DB919F'), + BigInt('0x269B24CA6B12F26D'), + BigInt('0x59F5D4025B277B14'), + BigInt('0xD846C55A0B79E09F'), + BigInt('0xA72835923B4C69E6'), + BigInt('0xEFF9C1B9F35344E2'), + BigInt('0x90973171C366CD9B'), + BigInt('0x1124202993385610'), + BigInt('0x6E4AD0E1A30DDF69'), + BigInt('0x8087C87E03060C18'), + BigInt('0xFFE938B633338561'), + BigInt('0x7E5A29EE636D1EEA'), + BigInt('0x0134D92653589793'), + BigInt('0x49E52D0D9B47BA97'), + BigInt('0x368BDDC5AB7233EE'), + BigInt('0xB738CC9DFB2CA865'), + BigInt('0xC8563C55CB19211C'), + BigInt('0x5E7BDBF1E3AC9DEC'), + BigInt('0x21152B39D3991495'), + BigInt('0xA0A63A6183C78F1E'), + BigInt('0xDFC8CAA9B3F20667'), + BigInt('0x97193E827BED2B63'), + BigInt('0xE877CE4A4BD8A21A'), + BigInt('0x69C4DF121B863991'), + BigInt('0x16AA2FDA2BB3B0E8'), + BigInt('0xF86737458BB86399'), + BigInt('0x8709C78DBB8DEAE0'), + BigInt('0x06BAD6D5EBD3716B'), + BigInt('0x79D4261DDBE6F812'), + BigInt('0x3105D23613F9D516'), + BigInt('0x4E6B22FE23CC5C6F'), + BigInt('0xCFD833A67392C7E4'), + BigInt('0xB0B6C36E43A74E9D'), + BigInt('0x9A6C9329AC4BC9B5'), + BigInt('0xE50263E19C7E40CC'), + BigInt('0x64B172B9CC20DB47'), + BigInt('0x1BDF8271FC15523E'), + BigInt('0x530E765A340A7F3A'), + BigInt('0x2C608692043FF643'), + BigInt('0xADD397CA54616DC8'), + BigInt('0xD2BD67026454E4B1'), + BigInt('0x3C707F9DC45F37C0'), + BigInt('0x431E8F55F46ABEB9'), + BigInt('0xC2AD9E0DA4342532'), + BigInt('0xBDC36EC59401AC4B'), + BigInt('0xF5129AEE5C1E814F'), + BigInt('0x8A7C6A266C2B0836'), + BigInt('0x0BCF7B7E3C7593BD'), + BigInt('0x74A18BB60C401AC4'), + BigInt('0xE28C6C1224F5A634'), + BigInt('0x9DE29CDA14C02F4D'), + BigInt('0x1C518D82449EB4C6'), + BigInt('0x633F7D4A74AB3DBF'), + BigInt('0x2BEE8961BCB410BB'), + BigInt('0x548079A98C8199C2'), + BigInt('0xD53368F1DCDF0249'), + BigInt('0xAA5D9839ECEA8B30'), + BigInt('0x449080A64CE15841'), + BigInt('0x3BFE706E7CD4D138'), + BigInt('0xBA4D61362C8A4AB3'), + BigInt('0xC52391FE1CBFC3CA'), + BigInt('0x8DF265D5D4A0EECE'), + BigInt('0xF29C951DE49567B7'), + BigInt('0x732F8445B4CBFC3C'), + BigInt('0x0C41748D84FE7545'), + BigInt('0x6BAD6D5EBD3716B7'), + BigInt('0x14C39D968D029FCE'), + BigInt('0x95708CCEDD5C0445'), + BigInt('0xEA1E7C06ED698D3C'), + BigInt('0xA2CF882D2576A038'), + BigInt('0xDDA178E515432941'), + BigInt('0x5C1269BD451DB2CA'), + BigInt('0x237C997575283BB3'), + BigInt('0xCDB181EAD523E8C2'), + BigInt('0xB2DF7122E51661BB'), + BigInt('0x336C607AB548FA30'), + BigInt('0x4C0290B2857D7349'), + BigInt('0x04D364994D625E4D'), + BigInt('0x7BBD94517D57D734'), + BigInt('0xFA0E85092D094CBF'), + BigInt('0x856075C11D3CC5C6'), + BigInt('0x134D926535897936'), + BigInt('0x6C2362AD05BCF04F'), + BigInt('0xED9073F555E26BC4'), + BigInt('0x92FE833D65D7E2BD'), + BigInt('0xDA2F7716ADC8CFB9'), + BigInt('0xA54187DE9DFD46C0'), + BigInt('0x24F29686CDA3DD4B'), + BigInt('0x5B9C664EFD965432'), + BigInt('0xB5517ED15D9D8743'), + BigInt('0xCA3F8E196DA80E3A'), + BigInt('0x4B8C9F413DF695B1'), + BigInt('0x34E26F890DC31CC8'), + BigInt('0x7C339BA2C5DC31CC'), + BigInt('0x035D6B6AF5E9B8B5'), + BigInt('0x82EE7A32A5B7233E'), + BigInt('0xFD808AFA9582AA47'), + BigInt('0x4D364994D625E4DA'), + BigInt('0x3258B95CE6106DA3'), + BigInt('0xB3EBA804B64EF628'), + BigInt('0xCC8558CC867B7F51'), + BigInt('0x8454ACE74E645255'), + BigInt('0xFB3A5C2F7E51DB2C'), + BigInt('0x7A894D772E0F40A7'), + BigInt('0x05E7BDBF1E3AC9DE'), + BigInt('0xEB2AA520BE311AAF'), + BigInt('0x944455E88E0493D6'), + BigInt('0x15F744B0DE5A085D'), + BigInt('0x6A99B478EE6F8124'), + BigInt('0x224840532670AC20'), + BigInt('0x5D26B09B16452559'), + BigInt('0xDC95A1C3461BBED2'), + BigInt('0xA3FB510B762E37AB'), + BigInt('0x35D6B6AF5E9B8B5B'), + BigInt('0x4AB846676EAE0222'), + BigInt('0xCB0B573F3EF099A9'), + BigInt('0xB465A7F70EC510D0'), + BigInt('0xFCB453DCC6DA3DD4'), + BigInt('0x83DAA314F6EFB4AD'), + BigInt('0x0269B24CA6B12F26'), + BigInt('0x7D0742849684A65F'), + BigInt('0x93CA5A1B368F752E'), + BigInt('0xECA4AAD306BAFC57'), + BigInt('0x6D17BB8B56E467DC'), + BigInt('0x12794B4366D1EEA5'), + BigInt('0x5AA8BF68AECEC3A1'), + BigInt('0x25C64FA09EFB4AD8'), + BigInt('0xA4755EF8CEA5D153'), + BigInt('0xDB1BAE30FE90582A'), + BigInt('0xBCF7B7E3C7593BD8'), + BigInt('0xC399472BF76CB2A1'), + BigInt('0x422A5673A732292A'), + BigInt('0x3D44A6BB9707A053'), + BigInt('0x759552905F188D57'), + BigInt('0x0AFBA2586F2D042E'), + BigInt('0x8B48B3003F739FA5'), + BigInt('0xF42643C80F4616DC'), + BigInt('0x1AEB5B57AF4DC5AD'), + BigInt('0x6585AB9F9F784CD4'), + BigInt('0xE436BAC7CF26D75F'), + BigInt('0x9B584A0FFF135E26'), + BigInt('0xD389BE24370C7322'), + BigInt('0xACE74EEC0739FA5B'), + BigInt('0x2D545FB4576761D0'), + BigInt('0x523AAF7C6752E8A9'), + BigInt('0xC41748D84FE75459'), + BigInt('0xBB79B8107FD2DD20'), + BigInt('0x3ACAA9482F8C46AB'), + BigInt('0x45A459801FB9CFD2'), + BigInt('0x0D75ADABD7A6E2D6'), + BigInt('0x721B5D63E7936BAF'), + BigInt('0xF3A84C3BB7CDF024'), + BigInt('0x8CC6BCF387F8795D'), + BigInt('0x620BA46C27F3AA2C'), + BigInt('0x1D6554A417C62355'), + BigInt('0x9CD645FC4798B8DE'), + BigInt('0xE3B8B53477AD31A7'), + BigInt('0xAB69411FBFB21CA3'), + BigInt('0xD407B1D78F8795DA'), + BigInt('0x55B4A08FDFD90E51'), + BigInt('0x2ADA5047EFEC8728') + ] + + constructor() { + this._crc = BigInt(0) + } + + update(data: Buffer | string): void { + const buffer = typeof data === 'string' ? Buffer.from(data) : data + let crc = CRC64.flip64Bits(this._crc) + + for (const dataByte of buffer) { + const crcByte = Number(crc & BigInt(0xff)) + crc = this._table[crcByte ^ dataByte] ^ (crc >> BigInt(8)) + } + + this._crc = CRC64.flip64Bits(crc) + } + + digest(): string { + return this._crc.toString(16).toUpperCase() + } + + static flip64Bits(n: bigint): bigint { + return (BigInt(1) << BigInt(64)) - BigInt(1) - n + } +} + +export default CRC64 From fccc5ee6e648101244fb9c196bbfc9b9944cec39 Mon Sep 17 00:00:00 2001 From: Rob Herley Date: Tue, 26 Apr 2022 17:31:17 +0000 Subject: [PATCH 04/10] crc: move tbl out of class, more tests --- packages/artifact/__tests__/crc64.test.ts | 25 +- packages/artifact/src/internal/crc64.ts | 521 +++++++++++----------- 2 files changed, 284 insertions(+), 262 deletions(-) diff --git a/packages/artifact/__tests__/crc64.test.ts b/packages/artifact/__tests__/crc64.test.ts index 6f5a2e03a0..e27a43464b 100644 --- a/packages/artifact/__tests__/crc64.test.ts +++ b/packages/artifact/__tests__/crc64.test.ts @@ -2,8 +2,8 @@ import CRC64 from '../src/internal/crc64' const fixtures = { data: - '🚀 👉😎👉 Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.\nUt enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.\nDuis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.\nExcepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.', - expected: 'D170FCD035F4D958' + '🚀 👉😎👉 Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.\nUt enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.\nDuis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.\nExcepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.\n', + expected: '846CE4ADAD6223ED' } describe('@actions/artifact/src/internal/crc64', () => { @@ -19,4 +19,25 @@ describe('@actions/artifact/src/internal/crc64', () => { crc.update(buf) expect(crc.digest()).toEqual(fixtures.expected) }) + + it('CRC64 from split data', async () => { + const crc = new CRC64() + const splits = fixtures.data.split('\n').slice(0, -1) + for (const split of splits) { + crc.update(`${split}\n`) + } + expect(crc.digest()).toEqual(fixtures.expected) + }) + + it('flips 64 bits', async () => { + const tests = [ + [BigInt(0), BigInt('0xffffffffffffffff')], + [BigInt('0xffffffffffffffff'), BigInt(0)], + [BigInt('0xdeadbeef'), BigInt('0xffffffff21524110')] + ] + + for (const [input, expected] of tests) { + expect(CRC64.flip64Bits(input)).toEqual(expected) + } + }) }) diff --git a/packages/artifact/src/internal/crc64.ts b/packages/artifact/src/internal/crc64.ts index 97fcc3df26..1749601e89 100644 --- a/packages/artifact/src/internal/crc64.ts +++ b/packages/artifact/src/internal/crc64.ts @@ -9,267 +9,268 @@ * is used for Azure Storage: https://github.com/Azure/azure-storage-net */ +// when transpile target is >= ES2020 (after dropping node 12) these can be changed to bigint literals - ts(2737) +const AZURE_TABLE = [ + BigInt('0x0000000000000000'), + BigInt('0x7F6EF0C830358979'), + BigInt('0xFEDDE190606B12F2'), + BigInt('0x81B31158505E9B8B'), + BigInt('0xC962E5739841B68F'), + BigInt('0xB60C15BBA8743FF6'), + BigInt('0x37BF04E3F82AA47D'), + BigInt('0x48D1F42BC81F2D04'), + BigInt('0xA61CECB46814FE75'), + BigInt('0xD9721C7C5821770C'), + BigInt('0x58C10D24087FEC87'), + BigInt('0x27AFFDEC384A65FE'), + BigInt('0x6F7E09C7F05548FA'), + BigInt('0x1010F90FC060C183'), + BigInt('0x91A3E857903E5A08'), + BigInt('0xEECD189FA00BD371'), + BigInt('0x78E0FF3B88BE6F81'), + BigInt('0x078E0FF3B88BE6F8'), + BigInt('0x863D1EABE8D57D73'), + BigInt('0xF953EE63D8E0F40A'), + BigInt('0xB1821A4810FFD90E'), + BigInt('0xCEECEA8020CA5077'), + BigInt('0x4F5FFBD87094CBFC'), + BigInt('0x30310B1040A14285'), + BigInt('0xDEFC138FE0AA91F4'), + BigInt('0xA192E347D09F188D'), + BigInt('0x2021F21F80C18306'), + BigInt('0x5F4F02D7B0F40A7F'), + BigInt('0x179EF6FC78EB277B'), + BigInt('0x68F0063448DEAE02'), + BigInt('0xE943176C18803589'), + BigInt('0x962DE7A428B5BCF0'), + BigInt('0xF1C1FE77117CDF02'), + BigInt('0x8EAF0EBF2149567B'), + BigInt('0x0F1C1FE77117CDF0'), + BigInt('0x7072EF2F41224489'), + BigInt('0x38A31B04893D698D'), + BigInt('0x47CDEBCCB908E0F4'), + BigInt('0xC67EFA94E9567B7F'), + BigInt('0xB9100A5CD963F206'), + BigInt('0x57DD12C379682177'), + BigInt('0x28B3E20B495DA80E'), + BigInt('0xA900F35319033385'), + BigInt('0xD66E039B2936BAFC'), + BigInt('0x9EBFF7B0E12997F8'), + BigInt('0xE1D10778D11C1E81'), + BigInt('0x606216208142850A'), + BigInt('0x1F0CE6E8B1770C73'), + BigInt('0x8921014C99C2B083'), + BigInt('0xF64FF184A9F739FA'), + BigInt('0x77FCE0DCF9A9A271'), + BigInt('0x08921014C99C2B08'), + BigInt('0x4043E43F0183060C'), + BigInt('0x3F2D14F731B68F75'), + BigInt('0xBE9E05AF61E814FE'), + BigInt('0xC1F0F56751DD9D87'), + BigInt('0x2F3DEDF8F1D64EF6'), + BigInt('0x50531D30C1E3C78F'), + BigInt('0xD1E00C6891BD5C04'), + BigInt('0xAE8EFCA0A188D57D'), + BigInt('0xE65F088B6997F879'), + BigInt('0x9931F84359A27100'), + BigInt('0x1882E91B09FCEA8B'), + BigInt('0x67EC19D339C963F2'), + BigInt('0xD75ADABD7A6E2D6F'), + BigInt('0xA8342A754A5BA416'), + BigInt('0x29873B2D1A053F9D'), + BigInt('0x56E9CBE52A30B6E4'), + BigInt('0x1E383FCEE22F9BE0'), + BigInt('0x6156CF06D21A1299'), + BigInt('0xE0E5DE5E82448912'), + BigInt('0x9F8B2E96B271006B'), + BigInt('0x71463609127AD31A'), + BigInt('0x0E28C6C1224F5A63'), + BigInt('0x8F9BD7997211C1E8'), + BigInt('0xF0F5275142244891'), + BigInt('0xB824D37A8A3B6595'), + BigInt('0xC74A23B2BA0EECEC'), + BigInt('0x46F932EAEA507767'), + BigInt('0x3997C222DA65FE1E'), + BigInt('0xAFBA2586F2D042EE'), + BigInt('0xD0D4D54EC2E5CB97'), + BigInt('0x5167C41692BB501C'), + BigInt('0x2E0934DEA28ED965'), + BigInt('0x66D8C0F56A91F461'), + BigInt('0x19B6303D5AA47D18'), + BigInt('0x980521650AFAE693'), + BigInt('0xE76BD1AD3ACF6FEA'), + BigInt('0x09A6C9329AC4BC9B'), + BigInt('0x76C839FAAAF135E2'), + BigInt('0xF77B28A2FAAFAE69'), + BigInt('0x8815D86ACA9A2710'), + BigInt('0xC0C42C4102850A14'), + BigInt('0xBFAADC8932B0836D'), + BigInt('0x3E19CDD162EE18E6'), + BigInt('0x41773D1952DB919F'), + BigInt('0x269B24CA6B12F26D'), + BigInt('0x59F5D4025B277B14'), + BigInt('0xD846C55A0B79E09F'), + BigInt('0xA72835923B4C69E6'), + BigInt('0xEFF9C1B9F35344E2'), + BigInt('0x90973171C366CD9B'), + BigInt('0x1124202993385610'), + BigInt('0x6E4AD0E1A30DDF69'), + BigInt('0x8087C87E03060C18'), + BigInt('0xFFE938B633338561'), + BigInt('0x7E5A29EE636D1EEA'), + BigInt('0x0134D92653589793'), + BigInt('0x49E52D0D9B47BA97'), + BigInt('0x368BDDC5AB7233EE'), + BigInt('0xB738CC9DFB2CA865'), + BigInt('0xC8563C55CB19211C'), + BigInt('0x5E7BDBF1E3AC9DEC'), + BigInt('0x21152B39D3991495'), + BigInt('0xA0A63A6183C78F1E'), + BigInt('0xDFC8CAA9B3F20667'), + BigInt('0x97193E827BED2B63'), + BigInt('0xE877CE4A4BD8A21A'), + BigInt('0x69C4DF121B863991'), + BigInt('0x16AA2FDA2BB3B0E8'), + BigInt('0xF86737458BB86399'), + BigInt('0x8709C78DBB8DEAE0'), + BigInt('0x06BAD6D5EBD3716B'), + BigInt('0x79D4261DDBE6F812'), + BigInt('0x3105D23613F9D516'), + BigInt('0x4E6B22FE23CC5C6F'), + BigInt('0xCFD833A67392C7E4'), + BigInt('0xB0B6C36E43A74E9D'), + BigInt('0x9A6C9329AC4BC9B5'), + BigInt('0xE50263E19C7E40CC'), + BigInt('0x64B172B9CC20DB47'), + BigInt('0x1BDF8271FC15523E'), + BigInt('0x530E765A340A7F3A'), + BigInt('0x2C608692043FF643'), + BigInt('0xADD397CA54616DC8'), + BigInt('0xD2BD67026454E4B1'), + BigInt('0x3C707F9DC45F37C0'), + BigInt('0x431E8F55F46ABEB9'), + BigInt('0xC2AD9E0DA4342532'), + BigInt('0xBDC36EC59401AC4B'), + BigInt('0xF5129AEE5C1E814F'), + BigInt('0x8A7C6A266C2B0836'), + BigInt('0x0BCF7B7E3C7593BD'), + BigInt('0x74A18BB60C401AC4'), + BigInt('0xE28C6C1224F5A634'), + BigInt('0x9DE29CDA14C02F4D'), + BigInt('0x1C518D82449EB4C6'), + BigInt('0x633F7D4A74AB3DBF'), + BigInt('0x2BEE8961BCB410BB'), + BigInt('0x548079A98C8199C2'), + BigInt('0xD53368F1DCDF0249'), + BigInt('0xAA5D9839ECEA8B30'), + BigInt('0x449080A64CE15841'), + BigInt('0x3BFE706E7CD4D138'), + BigInt('0xBA4D61362C8A4AB3'), + BigInt('0xC52391FE1CBFC3CA'), + BigInt('0x8DF265D5D4A0EECE'), + BigInt('0xF29C951DE49567B7'), + BigInt('0x732F8445B4CBFC3C'), + BigInt('0x0C41748D84FE7545'), + BigInt('0x6BAD6D5EBD3716B7'), + BigInt('0x14C39D968D029FCE'), + BigInt('0x95708CCEDD5C0445'), + BigInt('0xEA1E7C06ED698D3C'), + BigInt('0xA2CF882D2576A038'), + BigInt('0xDDA178E515432941'), + BigInt('0x5C1269BD451DB2CA'), + BigInt('0x237C997575283BB3'), + BigInt('0xCDB181EAD523E8C2'), + BigInt('0xB2DF7122E51661BB'), + BigInt('0x336C607AB548FA30'), + BigInt('0x4C0290B2857D7349'), + BigInt('0x04D364994D625E4D'), + BigInt('0x7BBD94517D57D734'), + BigInt('0xFA0E85092D094CBF'), + BigInt('0x856075C11D3CC5C6'), + BigInt('0x134D926535897936'), + BigInt('0x6C2362AD05BCF04F'), + BigInt('0xED9073F555E26BC4'), + BigInt('0x92FE833D65D7E2BD'), + BigInt('0xDA2F7716ADC8CFB9'), + BigInt('0xA54187DE9DFD46C0'), + BigInt('0x24F29686CDA3DD4B'), + BigInt('0x5B9C664EFD965432'), + BigInt('0xB5517ED15D9D8743'), + BigInt('0xCA3F8E196DA80E3A'), + BigInt('0x4B8C9F413DF695B1'), + BigInt('0x34E26F890DC31CC8'), + BigInt('0x7C339BA2C5DC31CC'), + BigInt('0x035D6B6AF5E9B8B5'), + BigInt('0x82EE7A32A5B7233E'), + BigInt('0xFD808AFA9582AA47'), + BigInt('0x4D364994D625E4DA'), + BigInt('0x3258B95CE6106DA3'), + BigInt('0xB3EBA804B64EF628'), + BigInt('0xCC8558CC867B7F51'), + BigInt('0x8454ACE74E645255'), + BigInt('0xFB3A5C2F7E51DB2C'), + BigInt('0x7A894D772E0F40A7'), + BigInt('0x05E7BDBF1E3AC9DE'), + BigInt('0xEB2AA520BE311AAF'), + BigInt('0x944455E88E0493D6'), + BigInt('0x15F744B0DE5A085D'), + BigInt('0x6A99B478EE6F8124'), + BigInt('0x224840532670AC20'), + BigInt('0x5D26B09B16452559'), + BigInt('0xDC95A1C3461BBED2'), + BigInt('0xA3FB510B762E37AB'), + BigInt('0x35D6B6AF5E9B8B5B'), + BigInt('0x4AB846676EAE0222'), + BigInt('0xCB0B573F3EF099A9'), + BigInt('0xB465A7F70EC510D0'), + BigInt('0xFCB453DCC6DA3DD4'), + BigInt('0x83DAA314F6EFB4AD'), + BigInt('0x0269B24CA6B12F26'), + BigInt('0x7D0742849684A65F'), + BigInt('0x93CA5A1B368F752E'), + BigInt('0xECA4AAD306BAFC57'), + BigInt('0x6D17BB8B56E467DC'), + BigInt('0x12794B4366D1EEA5'), + BigInt('0x5AA8BF68AECEC3A1'), + BigInt('0x25C64FA09EFB4AD8'), + BigInt('0xA4755EF8CEA5D153'), + BigInt('0xDB1BAE30FE90582A'), + BigInt('0xBCF7B7E3C7593BD8'), + BigInt('0xC399472BF76CB2A1'), + BigInt('0x422A5673A732292A'), + BigInt('0x3D44A6BB9707A053'), + BigInt('0x759552905F188D57'), + BigInt('0x0AFBA2586F2D042E'), + BigInt('0x8B48B3003F739FA5'), + BigInt('0xF42643C80F4616DC'), + BigInt('0x1AEB5B57AF4DC5AD'), + BigInt('0x6585AB9F9F784CD4'), + BigInt('0xE436BAC7CF26D75F'), + BigInt('0x9B584A0FFF135E26'), + BigInt('0xD389BE24370C7322'), + BigInt('0xACE74EEC0739FA5B'), + BigInt('0x2D545FB4576761D0'), + BigInt('0x523AAF7C6752E8A9'), + BigInt('0xC41748D84FE75459'), + BigInt('0xBB79B8107FD2DD20'), + BigInt('0x3ACAA9482F8C46AB'), + BigInt('0x45A459801FB9CFD2'), + BigInt('0x0D75ADABD7A6E2D6'), + BigInt('0x721B5D63E7936BAF'), + BigInt('0xF3A84C3BB7CDF024'), + BigInt('0x8CC6BCF387F8795D'), + BigInt('0x620BA46C27F3AA2C'), + BigInt('0x1D6554A417C62355'), + BigInt('0x9CD645FC4798B8DE'), + BigInt('0xE3B8B53477AD31A7'), + BigInt('0xAB69411FBFB21CA3'), + BigInt('0xD407B1D78F8795DA'), + BigInt('0x55B4A08FDFD90E51'), + BigInt('0x2ADA5047EFEC8728') +] + class CRC64 { private _crc: bigint - // when transpile target is >= ES2020 (after dropping node 12) these can be changed to bigint literals - ts(2737) - private _table = [ - BigInt('0x0000000000000000'), - BigInt('0x7F6EF0C830358979'), - BigInt('0xFEDDE190606B12F2'), - BigInt('0x81B31158505E9B8B'), - BigInt('0xC962E5739841B68F'), - BigInt('0xB60C15BBA8743FF6'), - BigInt('0x37BF04E3F82AA47D'), - BigInt('0x48D1F42BC81F2D04'), - BigInt('0xA61CECB46814FE75'), - BigInt('0xD9721C7C5821770C'), - BigInt('0x58C10D24087FEC87'), - BigInt('0x27AFFDEC384A65FE'), - BigInt('0x6F7E09C7F05548FA'), - BigInt('0x1010F90FC060C183'), - BigInt('0x91A3E857903E5A08'), - BigInt('0xEECD189FA00BD371'), - BigInt('0x78E0FF3B88BE6F81'), - BigInt('0x078E0FF3B88BE6F8'), - BigInt('0x863D1EABE8D57D73'), - BigInt('0xF953EE63D8E0F40A'), - BigInt('0xB1821A4810FFD90E'), - BigInt('0xCEECEA8020CA5077'), - BigInt('0x4F5FFBD87094CBFC'), - BigInt('0x30310B1040A14285'), - BigInt('0xDEFC138FE0AA91F4'), - BigInt('0xA192E347D09F188D'), - BigInt('0x2021F21F80C18306'), - BigInt('0x5F4F02D7B0F40A7F'), - BigInt('0x179EF6FC78EB277B'), - BigInt('0x68F0063448DEAE02'), - BigInt('0xE943176C18803589'), - BigInt('0x962DE7A428B5BCF0'), - BigInt('0xF1C1FE77117CDF02'), - BigInt('0x8EAF0EBF2149567B'), - BigInt('0x0F1C1FE77117CDF0'), - BigInt('0x7072EF2F41224489'), - BigInt('0x38A31B04893D698D'), - BigInt('0x47CDEBCCB908E0F4'), - BigInt('0xC67EFA94E9567B7F'), - BigInt('0xB9100A5CD963F206'), - BigInt('0x57DD12C379682177'), - BigInt('0x28B3E20B495DA80E'), - BigInt('0xA900F35319033385'), - BigInt('0xD66E039B2936BAFC'), - BigInt('0x9EBFF7B0E12997F8'), - BigInt('0xE1D10778D11C1E81'), - BigInt('0x606216208142850A'), - BigInt('0x1F0CE6E8B1770C73'), - BigInt('0x8921014C99C2B083'), - BigInt('0xF64FF184A9F739FA'), - BigInt('0x77FCE0DCF9A9A271'), - BigInt('0x08921014C99C2B08'), - BigInt('0x4043E43F0183060C'), - BigInt('0x3F2D14F731B68F75'), - BigInt('0xBE9E05AF61E814FE'), - BigInt('0xC1F0F56751DD9D87'), - BigInt('0x2F3DEDF8F1D64EF6'), - BigInt('0x50531D30C1E3C78F'), - BigInt('0xD1E00C6891BD5C04'), - BigInt('0xAE8EFCA0A188D57D'), - BigInt('0xE65F088B6997F879'), - BigInt('0x9931F84359A27100'), - BigInt('0x1882E91B09FCEA8B'), - BigInt('0x67EC19D339C963F2'), - BigInt('0xD75ADABD7A6E2D6F'), - BigInt('0xA8342A754A5BA416'), - BigInt('0x29873B2D1A053F9D'), - BigInt('0x56E9CBE52A30B6E4'), - BigInt('0x1E383FCEE22F9BE0'), - BigInt('0x6156CF06D21A1299'), - BigInt('0xE0E5DE5E82448912'), - BigInt('0x9F8B2E96B271006B'), - BigInt('0x71463609127AD31A'), - BigInt('0x0E28C6C1224F5A63'), - BigInt('0x8F9BD7997211C1E8'), - BigInt('0xF0F5275142244891'), - BigInt('0xB824D37A8A3B6595'), - BigInt('0xC74A23B2BA0EECEC'), - BigInt('0x46F932EAEA507767'), - BigInt('0x3997C222DA65FE1E'), - BigInt('0xAFBA2586F2D042EE'), - BigInt('0xD0D4D54EC2E5CB97'), - BigInt('0x5167C41692BB501C'), - BigInt('0x2E0934DEA28ED965'), - BigInt('0x66D8C0F56A91F461'), - BigInt('0x19B6303D5AA47D18'), - BigInt('0x980521650AFAE693'), - BigInt('0xE76BD1AD3ACF6FEA'), - BigInt('0x09A6C9329AC4BC9B'), - BigInt('0x76C839FAAAF135E2'), - BigInt('0xF77B28A2FAAFAE69'), - BigInt('0x8815D86ACA9A2710'), - BigInt('0xC0C42C4102850A14'), - BigInt('0xBFAADC8932B0836D'), - BigInt('0x3E19CDD162EE18E6'), - BigInt('0x41773D1952DB919F'), - BigInt('0x269B24CA6B12F26D'), - BigInt('0x59F5D4025B277B14'), - BigInt('0xD846C55A0B79E09F'), - BigInt('0xA72835923B4C69E6'), - BigInt('0xEFF9C1B9F35344E2'), - BigInt('0x90973171C366CD9B'), - BigInt('0x1124202993385610'), - BigInt('0x6E4AD0E1A30DDF69'), - BigInt('0x8087C87E03060C18'), - BigInt('0xFFE938B633338561'), - BigInt('0x7E5A29EE636D1EEA'), - BigInt('0x0134D92653589793'), - BigInt('0x49E52D0D9B47BA97'), - BigInt('0x368BDDC5AB7233EE'), - BigInt('0xB738CC9DFB2CA865'), - BigInt('0xC8563C55CB19211C'), - BigInt('0x5E7BDBF1E3AC9DEC'), - BigInt('0x21152B39D3991495'), - BigInt('0xA0A63A6183C78F1E'), - BigInt('0xDFC8CAA9B3F20667'), - BigInt('0x97193E827BED2B63'), - BigInt('0xE877CE4A4BD8A21A'), - BigInt('0x69C4DF121B863991'), - BigInt('0x16AA2FDA2BB3B0E8'), - BigInt('0xF86737458BB86399'), - BigInt('0x8709C78DBB8DEAE0'), - BigInt('0x06BAD6D5EBD3716B'), - BigInt('0x79D4261DDBE6F812'), - BigInt('0x3105D23613F9D516'), - BigInt('0x4E6B22FE23CC5C6F'), - BigInt('0xCFD833A67392C7E4'), - BigInt('0xB0B6C36E43A74E9D'), - BigInt('0x9A6C9329AC4BC9B5'), - BigInt('0xE50263E19C7E40CC'), - BigInt('0x64B172B9CC20DB47'), - BigInt('0x1BDF8271FC15523E'), - BigInt('0x530E765A340A7F3A'), - BigInt('0x2C608692043FF643'), - BigInt('0xADD397CA54616DC8'), - BigInt('0xD2BD67026454E4B1'), - BigInt('0x3C707F9DC45F37C0'), - BigInt('0x431E8F55F46ABEB9'), - BigInt('0xC2AD9E0DA4342532'), - BigInt('0xBDC36EC59401AC4B'), - BigInt('0xF5129AEE5C1E814F'), - BigInt('0x8A7C6A266C2B0836'), - BigInt('0x0BCF7B7E3C7593BD'), - BigInt('0x74A18BB60C401AC4'), - BigInt('0xE28C6C1224F5A634'), - BigInt('0x9DE29CDA14C02F4D'), - BigInt('0x1C518D82449EB4C6'), - BigInt('0x633F7D4A74AB3DBF'), - BigInt('0x2BEE8961BCB410BB'), - BigInt('0x548079A98C8199C2'), - BigInt('0xD53368F1DCDF0249'), - BigInt('0xAA5D9839ECEA8B30'), - BigInt('0x449080A64CE15841'), - BigInt('0x3BFE706E7CD4D138'), - BigInt('0xBA4D61362C8A4AB3'), - BigInt('0xC52391FE1CBFC3CA'), - BigInt('0x8DF265D5D4A0EECE'), - BigInt('0xF29C951DE49567B7'), - BigInt('0x732F8445B4CBFC3C'), - BigInt('0x0C41748D84FE7545'), - BigInt('0x6BAD6D5EBD3716B7'), - BigInt('0x14C39D968D029FCE'), - BigInt('0x95708CCEDD5C0445'), - BigInt('0xEA1E7C06ED698D3C'), - BigInt('0xA2CF882D2576A038'), - BigInt('0xDDA178E515432941'), - BigInt('0x5C1269BD451DB2CA'), - BigInt('0x237C997575283BB3'), - BigInt('0xCDB181EAD523E8C2'), - BigInt('0xB2DF7122E51661BB'), - BigInt('0x336C607AB548FA30'), - BigInt('0x4C0290B2857D7349'), - BigInt('0x04D364994D625E4D'), - BigInt('0x7BBD94517D57D734'), - BigInt('0xFA0E85092D094CBF'), - BigInt('0x856075C11D3CC5C6'), - BigInt('0x134D926535897936'), - BigInt('0x6C2362AD05BCF04F'), - BigInt('0xED9073F555E26BC4'), - BigInt('0x92FE833D65D7E2BD'), - BigInt('0xDA2F7716ADC8CFB9'), - BigInt('0xA54187DE9DFD46C0'), - BigInt('0x24F29686CDA3DD4B'), - BigInt('0x5B9C664EFD965432'), - BigInt('0xB5517ED15D9D8743'), - BigInt('0xCA3F8E196DA80E3A'), - BigInt('0x4B8C9F413DF695B1'), - BigInt('0x34E26F890DC31CC8'), - BigInt('0x7C339BA2C5DC31CC'), - BigInt('0x035D6B6AF5E9B8B5'), - BigInt('0x82EE7A32A5B7233E'), - BigInt('0xFD808AFA9582AA47'), - BigInt('0x4D364994D625E4DA'), - BigInt('0x3258B95CE6106DA3'), - BigInt('0xB3EBA804B64EF628'), - BigInt('0xCC8558CC867B7F51'), - BigInt('0x8454ACE74E645255'), - BigInt('0xFB3A5C2F7E51DB2C'), - BigInt('0x7A894D772E0F40A7'), - BigInt('0x05E7BDBF1E3AC9DE'), - BigInt('0xEB2AA520BE311AAF'), - BigInt('0x944455E88E0493D6'), - BigInt('0x15F744B0DE5A085D'), - BigInt('0x6A99B478EE6F8124'), - BigInt('0x224840532670AC20'), - BigInt('0x5D26B09B16452559'), - BigInt('0xDC95A1C3461BBED2'), - BigInt('0xA3FB510B762E37AB'), - BigInt('0x35D6B6AF5E9B8B5B'), - BigInt('0x4AB846676EAE0222'), - BigInt('0xCB0B573F3EF099A9'), - BigInt('0xB465A7F70EC510D0'), - BigInt('0xFCB453DCC6DA3DD4'), - BigInt('0x83DAA314F6EFB4AD'), - BigInt('0x0269B24CA6B12F26'), - BigInt('0x7D0742849684A65F'), - BigInt('0x93CA5A1B368F752E'), - BigInt('0xECA4AAD306BAFC57'), - BigInt('0x6D17BB8B56E467DC'), - BigInt('0x12794B4366D1EEA5'), - BigInt('0x5AA8BF68AECEC3A1'), - BigInt('0x25C64FA09EFB4AD8'), - BigInt('0xA4755EF8CEA5D153'), - BigInt('0xDB1BAE30FE90582A'), - BigInt('0xBCF7B7E3C7593BD8'), - BigInt('0xC399472BF76CB2A1'), - BigInt('0x422A5673A732292A'), - BigInt('0x3D44A6BB9707A053'), - BigInt('0x759552905F188D57'), - BigInt('0x0AFBA2586F2D042E'), - BigInt('0x8B48B3003F739FA5'), - BigInt('0xF42643C80F4616DC'), - BigInt('0x1AEB5B57AF4DC5AD'), - BigInt('0x6585AB9F9F784CD4'), - BigInt('0xE436BAC7CF26D75F'), - BigInt('0x9B584A0FFF135E26'), - BigInt('0xD389BE24370C7322'), - BigInt('0xACE74EEC0739FA5B'), - BigInt('0x2D545FB4576761D0'), - BigInt('0x523AAF7C6752E8A9'), - BigInt('0xC41748D84FE75459'), - BigInt('0xBB79B8107FD2DD20'), - BigInt('0x3ACAA9482F8C46AB'), - BigInt('0x45A459801FB9CFD2'), - BigInt('0x0D75ADABD7A6E2D6'), - BigInt('0x721B5D63E7936BAF'), - BigInt('0xF3A84C3BB7CDF024'), - BigInt('0x8CC6BCF387F8795D'), - BigInt('0x620BA46C27F3AA2C'), - BigInt('0x1D6554A417C62355'), - BigInt('0x9CD645FC4798B8DE'), - BigInt('0xE3B8B53477AD31A7'), - BigInt('0xAB69411FBFB21CA3'), - BigInt('0xD407B1D78F8795DA'), - BigInt('0x55B4A08FDFD90E51'), - BigInt('0x2ADA5047EFEC8728') - ] constructor() { this._crc = BigInt(0) @@ -281,7 +282,7 @@ class CRC64 { for (const dataByte of buffer) { const crcByte = Number(crc & BigInt(0xff)) - crc = this._table[crcByte ^ dataByte] ^ (crc >> BigInt(8)) + crc = AZURE_TABLE[crcByte ^ dataByte] ^ (crc >> BigInt(8)) } this._crc = CRC64.flip64Bits(crc) From 3f95e2ea4fe0dd73aaebdcd0f81fbc6fef7d2c4e Mon Sep 17 00:00:00 2001 From: Rob Herley Date: Tue, 26 Apr 2022 22:59:56 +0000 Subject: [PATCH 05/10] crc: update headers & digest stream --- packages/artifact/__tests__/util.test.ts | 6 ++---- packages/artifact/src/internal/utils.ts | 10 ++++------ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/packages/artifact/__tests__/util.test.ts b/packages/artifact/__tests__/util.test.ts index 73e3c78c63..22af67813b 100644 --- a/packages/artifact/__tests__/util.test.ts +++ b/packages/artifact/__tests__/util.test.ts @@ -97,7 +97,7 @@ describe('Utils', () => { expect(headers['x-tfs-filelength']).toEqual(uncompressedLength) expect(headers['Content-Length']).toEqual(size) expect(headers['Content-Range']).toEqual(range) - expect(headers['X-Digest']).toEqual(`sha-256=${digest}`) + expect(headers['X-GH-Actions-CRC64']).toEqual(digest) }) it('Test constructing upload headers with only required parameter', () => { @@ -230,8 +230,6 @@ describe('Utils', () => { const stream = Readable.from(data) const digest = await utils.digestForStream(stream) - expect(digest).toBe( - '5e2bf57d3f40c4b6df69daf1936cb766f832374b4fc0259a7cbff06e2f70f269' - ) + expect(digest).toBe('FFFCD6894DC82C6D') }) }) diff --git a/packages/artifact/src/internal/utils.ts b/packages/artifact/src/internal/utils.ts index e23056f64e..46f3f245fb 100644 --- a/packages/artifact/src/internal/utils.ts +++ b/packages/artifact/src/internal/utils.ts @@ -1,4 +1,3 @@ -import * as crypto from 'crypto' import {promises as fs} from 'fs' import {IncomingHttpHeaders} from 'http' import {debug, info, warning} from '@actions/core' @@ -12,6 +11,7 @@ import { getRetryMultiplier, getInitialRetryIntervalInMilliseconds } from './config-variables' +import CRC64 from './crc64' /** * Returns a retry time in milliseconds that exponentially gets larger @@ -205,8 +205,7 @@ export function getUploadHeaders( requestOptions['Content-Range'] = contentRange } if (digest) { - // TODO(robherley): should we use 'Digest' directly? https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Digest - requestOptions['X-Digest'] = `sha-256=${digest}` + requestOptions['X-GH-Actions-CRC64'] = digest } return requestOptions @@ -302,10 +301,9 @@ export async function digestForStream( stream: NodeJS.ReadableStream ): Promise { return new Promise((resolve, reject) => { - // TODO(robherley): switch to crc64 for production - const hasher = crypto.createHash('sha256') + const hasher = new CRC64() stream.on('data', data => hasher.update(data)) - stream.on('end', () => resolve(hasher.digest('hex'))) + stream.on('end', () => resolve(hasher.digest())) stream.on('error', reject) }) } From 9e285cc3fa6ec6d66b7a91b812d8027dcdda712f Mon Sep 17 00:00:00 2001 From: Rob Herley Date: Tue, 26 Apr 2022 23:21:31 +0000 Subject: [PATCH 06/10] crc: update test header data --- packages/artifact/__tests__/util.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/artifact/__tests__/util.test.ts b/packages/artifact/__tests__/util.test.ts index 22af67813b..6804d418d7 100644 --- a/packages/artifact/__tests__/util.test.ts +++ b/packages/artifact/__tests__/util.test.ts @@ -75,8 +75,7 @@ describe('Utils', () => { const size = 24 const uncompressedLength = 100 const range = 'bytes 0-199/200' - const digest = - '5e2bf57d3f40c4b6df69daf1936cb766f832374b4fc0259a7cbff06e2f70f269' + const digest = 'FFFCD6894DC82C6D' const headers = utils.getUploadHeaders( contentType, true, From d5c547c19f56e89562cca8e464f29c05412f51e1 Mon Sep 17 00:00:00 2001 From: Rob Herley Date: Wed, 27 Apr 2022 22:22:35 +0000 Subject: [PATCH 07/10] update crc header key --- packages/artifact/__tests__/util.test.ts | 2 +- packages/artifact/src/internal/utils.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/artifact/__tests__/util.test.ts b/packages/artifact/__tests__/util.test.ts index 6804d418d7..938df3bb46 100644 --- a/packages/artifact/__tests__/util.test.ts +++ b/packages/artifact/__tests__/util.test.ts @@ -96,7 +96,7 @@ describe('Utils', () => { expect(headers['x-tfs-filelength']).toEqual(uncompressedLength) expect(headers['Content-Length']).toEqual(size) expect(headers['Content-Range']).toEqual(range) - expect(headers['X-GH-Actions-CRC64']).toEqual(digest) + expect(headers['x-actions-result-crc64']).toEqual(digest) }) it('Test constructing upload headers with only required parameter', () => { diff --git a/packages/artifact/src/internal/utils.ts b/packages/artifact/src/internal/utils.ts index 46f3f245fb..3fa2923e42 100644 --- a/packages/artifact/src/internal/utils.ts +++ b/packages/artifact/src/internal/utils.ts @@ -205,7 +205,7 @@ export function getUploadHeaders( requestOptions['Content-Range'] = contentRange } if (digest) { - requestOptions['X-GH-Actions-CRC64'] = digest + requestOptions['x-actions-result-crc64'] = digest } return requestOptions From b68735e06098244d8f6a11cfac4f0000d6807e4c Mon Sep 17 00:00:00 2001 From: Rob Herley Date: Wed, 27 Apr 2022 22:24:16 +0000 Subject: [PATCH 08/10] misc pr feedback --- packages/artifact/src/internal/crc64.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/artifact/src/internal/crc64.ts b/packages/artifact/src/internal/crc64.ts index 1749601e89..381e27abd6 100644 --- a/packages/artifact/src/internal/crc64.ts +++ b/packages/artifact/src/internal/crc64.ts @@ -6,11 +6,11 @@ * but without the slicing-by-8 optimization: https://cs.opensource.google/go/go/+/master:src/hash/crc64/crc64.go * * This implementation uses a pregenerated table based on 0x9A6C9329AC4BC9B5 as the polynomial, the same polynomial that - * is used for Azure Storage: https://github.com/Azure/azure-storage-net + * is used for Azure Storage: https://github.com/Azure/azure-storage-net/blob/cbe605f9faa01bfc3003d75fc5a16b2eaccfe102/Lib/Common/Core/Util/Crc64.cs#L27 */ // when transpile target is >= ES2020 (after dropping node 12) these can be changed to bigint literals - ts(2737) -const AZURE_TABLE = [ +const PREGEN_POLY_TABLE = [ BigInt('0x0000000000000000'), BigInt('0x7F6EF0C830358979'), BigInt('0xFEDDE190606B12F2'), @@ -282,7 +282,7 @@ class CRC64 { for (const dataByte of buffer) { const crcByte = Number(crc & BigInt(0xff)) - crc = AZURE_TABLE[crcByte ^ dataByte] ^ (crc >> BigInt(8)) + crc = PREGEN_POLY_TABLE[crcByte ^ dataByte] ^ (crc >> BigInt(8)) } this._crc = CRC64.flip64Bits(crc) From a3c696e88e8572ce57e213ed63921bdd2fdb211e Mon Sep 17 00:00:00 2001 From: Rob Herley Date: Thu, 5 May 2022 09:26:38 -0400 Subject: [PATCH 09/10] add md5 and use b64 for digest encodings --- packages/artifact/__tests__/crc64.test.ts | 24 ++++++++++++++---- packages/artifact/__tests__/util.test.ts | 13 +++++++--- packages/artifact/src/internal/crc64.ts | 21 +++++++++++++-- packages/artifact/src/internal/utils.ts | 31 ++++++++++++++++++----- 4 files changed, 71 insertions(+), 18 deletions(-) diff --git a/packages/artifact/__tests__/crc64.test.ts b/packages/artifact/__tests__/crc64.test.ts index e27a43464b..f5b504fa4a 100644 --- a/packages/artifact/__tests__/crc64.test.ts +++ b/packages/artifact/__tests__/crc64.test.ts @@ -1,23 +1,36 @@ -import CRC64 from '../src/internal/crc64' +import CRC64, {CRC64DigestEncoding} from '../src/internal/crc64' const fixtures = { data: '🚀 👉😎👉 Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.\nUt enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.\nDuis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.\nExcepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.\n', - expected: '846CE4ADAD6223ED' + expected: { + hex: '846CE4ADAD6223ED', + base64: '7SNira3kbIQ=', + buffer: Buffer.from([0xed, 0x23, 0x62, 0xad, 0xad, 0xe4, 0x6c, 0x84]) + } +} + +function assertEncodings(crc: CRC64): void { + const encodings = Object.keys(fixtures.expected) as CRC64DigestEncoding[] + for (const encoding of encodings) { + expect(crc.digest(encoding)).toEqual(fixtures.expected[encoding]) + } } describe('@actions/artifact/src/internal/crc64', () => { it('CRC64 from string', async () => { const crc = new CRC64() crc.update(fixtures.data) - expect(crc.digest()).toEqual(fixtures.expected) + + assertEncodings(crc) }) it('CRC64 from buffer', async () => { const crc = new CRC64() const buf = Buffer.from(fixtures.data) crc.update(buf) - expect(crc.digest()).toEqual(fixtures.expected) + + assertEncodings(crc) }) it('CRC64 from split data', async () => { @@ -26,7 +39,8 @@ describe('@actions/artifact/src/internal/crc64', () => { for (const split of splits) { crc.update(`${split}\n`) } - expect(crc.digest()).toEqual(fixtures.expected) + + assertEncodings(crc) }) it('flips 64 bits', async () => { diff --git a/packages/artifact/__tests__/util.test.ts b/packages/artifact/__tests__/util.test.ts index 938df3bb46..4d04384451 100644 --- a/packages/artifact/__tests__/util.test.ts +++ b/packages/artifact/__tests__/util.test.ts @@ -75,7 +75,10 @@ describe('Utils', () => { const size = 24 const uncompressedLength = 100 const range = 'bytes 0-199/200' - const digest = 'FFFCD6894DC82C6D' + const digest = { + crc64: 'bSzITYnW/P8=', + md5: 'Xiv1fT9AxLbfadrxk2y3ZvgyN0tPwCWafL/wbi9w8mk=' + } const headers = utils.getUploadHeaders( contentType, true, @@ -85,7 +88,7 @@ describe('Utils', () => { range, digest ) - expect(Object.keys(headers).length).toEqual(9) + expect(Object.keys(headers).length).toEqual(10) expect(headers['Accept']).toEqual( `application/json;api-version=${utils.getApiVersion()}` ) @@ -96,7 +99,8 @@ describe('Utils', () => { expect(headers['x-tfs-filelength']).toEqual(uncompressedLength) expect(headers['Content-Length']).toEqual(size) expect(headers['Content-Range']).toEqual(range) - expect(headers['x-actions-result-crc64']).toEqual(digest) + expect(headers['x-actions-results-crc64']).toEqual(digest.crc64) + expect(headers['x-actions-results-md5']).toEqual(digest.md5) }) it('Test constructing upload headers with only required parameter', () => { @@ -229,6 +233,7 @@ describe('Utils', () => { const stream = Readable.from(data) const digest = await utils.digestForStream(stream) - expect(digest).toBe('FFFCD6894DC82C6D') + expect(digest.crc64).toBe('bSzITYnW/P8=') + expect(digest.md5).toBe('Xiv1fT9AxLbfadrxk2y3ZvgyN0tPwCWafL/wbi9w8mk=') }) }) diff --git a/packages/artifact/src/internal/crc64.ts b/packages/artifact/src/internal/crc64.ts index 381e27abd6..452a828663 100644 --- a/packages/artifact/src/internal/crc64.ts +++ b/packages/artifact/src/internal/crc64.ts @@ -269,6 +269,8 @@ const PREGEN_POLY_TABLE = [ BigInt('0x2ADA5047EFEC8728') ] +export type CRC64DigestEncoding = 'hex' | 'base64' | 'buffer' + class CRC64 { private _crc: bigint @@ -288,8 +290,23 @@ class CRC64 { this._crc = CRC64.flip64Bits(crc) } - digest(): string { - return this._crc.toString(16).toUpperCase() + digest(encoding?: CRC64DigestEncoding): string | Buffer { + switch (encoding) { + case 'hex': + return this._crc.toString(16).toUpperCase() + case 'base64': + return this.toBuffer().toString('base64') + default: + return this.toBuffer() + } + } + + private toBuffer(): Buffer { + return Buffer.from( + [0, 8, 16, 24, 32, 40, 48, 56].map(s => + Number((this._crc >> BigInt(s)) & BigInt(0xff)) + ) + ) } static flip64Bits(n: bigint): bigint { diff --git a/packages/artifact/src/internal/utils.ts b/packages/artifact/src/internal/utils.ts index 3fa2923e42..94217a5c90 100644 --- a/packages/artifact/src/internal/utils.ts +++ b/packages/artifact/src/internal/utils.ts @@ -1,3 +1,4 @@ +import crypto from 'crypto' import {promises as fs} from 'fs' import {IncomingHttpHeaders} from 'http' import {debug, info, warning} from '@actions/core' @@ -182,7 +183,7 @@ export function getUploadHeaders( uncompressedLength?: number, contentLength?: number, contentRange?: string, - digest?: string + digest?: StreamDigest ): IHeaders { const requestOptions: IHeaders = {} requestOptions['Accept'] = `application/json;api-version=${getApiVersion()}` @@ -205,7 +206,8 @@ export function getUploadHeaders( requestOptions['Content-Range'] = contentRange } if (digest) { - requestOptions['x-actions-result-crc64'] = digest + requestOptions['x-actions-results-crc64'] = digest.crc64 + requestOptions['x-actions-results-md5'] = digest.md5 } return requestOptions @@ -297,13 +299,28 @@ export async function sleep(milliseconds: number): Promise { return new Promise(resolve => setTimeout(resolve, milliseconds)) } +export interface StreamDigest { + crc64: string + md5: string +} + export async function digestForStream( stream: NodeJS.ReadableStream -): Promise { +): Promise { return new Promise((resolve, reject) => { - const hasher = new CRC64() - stream.on('data', data => hasher.update(data)) - stream.on('end', () => resolve(hasher.digest())) - stream.on('error', reject) + const crc64 = new CRC64() + const md5 = crypto.createHash('sha256') + stream + .on('data', data => { + crc64.update(data) + md5.update(data) + }) + .on('end', () => + resolve({ + crc64: crc64.digest('base64') as string, + md5: md5.digest('base64') + }) + ) + .on('error', reject) }) } From 35e5aac5231f8befc21b9268780d51768f2d28b7 Mon Sep 17 00:00:00 2001 From: Rob Herley Date: Thu, 5 May 2022 10:32:09 -0400 Subject: [PATCH 10/10] actually use md5 not sha256 --- packages/artifact/__tests__/util.test.ts | 2 +- packages/artifact/src/internal/utils.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/artifact/__tests__/util.test.ts b/packages/artifact/__tests__/util.test.ts index 4d04384451..a62babba31 100644 --- a/packages/artifact/__tests__/util.test.ts +++ b/packages/artifact/__tests__/util.test.ts @@ -234,6 +234,6 @@ describe('Utils', () => { const digest = await utils.digestForStream(stream) expect(digest.crc64).toBe('bSzITYnW/P8=') - expect(digest.md5).toBe('Xiv1fT9AxLbfadrxk2y3ZvgyN0tPwCWafL/wbi9w8mk=') + expect(digest.md5).toBe('gKdR/eV3AoZAxBkADjPrpg==') }) }) diff --git a/packages/artifact/src/internal/utils.ts b/packages/artifact/src/internal/utils.ts index 94217a5c90..063b5dbfe7 100644 --- a/packages/artifact/src/internal/utils.ts +++ b/packages/artifact/src/internal/utils.ts @@ -309,7 +309,7 @@ export async function digestForStream( ): Promise { return new Promise((resolve, reject) => { const crc64 = new CRC64() - const md5 = crypto.createHash('sha256') + const md5 = crypto.createHash('md5') stream .on('data', data => { crc64.update(data)