From 993e284897d40e8119ca574fc80cd28c9ba5dcfd Mon Sep 17 00:00:00 2001 From: Morgan Zolob Date: Mon, 8 Aug 2022 03:15:02 -0700 Subject: [PATCH] @uppy/aws-s3-multipart: Fix race condition in `#uploadParts` (#3955) The `#uploadParts` function calls itself after any part is uploaded. It also determines which new chunks to upload based on their `state.busy` value. This introduced a race condition, as `state.busy` was being set to false in the XHR event handlers. So if one part were to complete while another part had finished the XHR request, but not yet completed, then an upload for that second part would be started again, despite the fact that the previous upload was still in progress. Multiple uploads for the same part at the same time cause numerous issues, and should never happen. This is especially noticeable when an XHR request fails. `#uploadPart` is wrapped in `#retryable`, so the part will be retried, however, for the entire `retryDelay`, the chunk's `state.busy` value would be false, meaning that if any other part completed, this part would be uploaded again, despite the fact that the upload is already ongoing. To fix this, this commit moves setting `state.busy` to the `before` and `after` functions of the `#retryable` call, so a part will remain `busy` for the entire time it is being uploaded/retried. --- .../aws-s3-multipart/src/MultipartUploader.js | 8 +- .../@uppy/aws-s3-multipart/src/index.test.js | 84 +++++++++++++++++-- 2 files changed, 81 insertions(+), 11 deletions(-) diff --git a/packages/@uppy/aws-s3-multipart/src/MultipartUploader.js b/packages/@uppy/aws-s3-multipart/src/MultipartUploader.js index 7bcc700b0c4..dc177d389cb 100644 --- a/packages/@uppy/aws-s3-multipart/src/MultipartUploader.js +++ b/packages/@uppy/aws-s3-multipart/src/MultipartUploader.js @@ -268,18 +268,18 @@ class MultipartUploader { #uploadPartRetryable (index, prePreparedPart) { return this.#retryable({ before: () => { + this.chunkState[index].busy = true this.partsInProgress += 1 }, attempt: () => this.#uploadPart(index, prePreparedPart), after: () => { + this.chunkState[index].busy = false this.partsInProgress -= 1 }, }) } #uploadPart (index, prePreparedPart) { - this.chunkState[index].busy = true - const valid = typeof prePreparedPart?.url === 'string' if (!valid) { throw new TypeError('AwsS3/Multipart: Got incorrect result for `prePreparedPart`, expected an object `{ url }`.') @@ -287,7 +287,6 @@ class MultipartUploader { const { url, headers } = prePreparedPart if (this.#aborted()) { - this.chunkState[index].busy = false throw createAbortError() } @@ -349,14 +348,12 @@ class MultipartUploader { xhr.addEventListener('abort', () => { cleanup() - this.chunkState[index].busy = false defer.reject(createAbortError()) }) xhr.addEventListener('load', (ev) => { cleanup() - this.chunkState[index].busy = false if (ev.target.status < 200 || ev.target.status >= 300) { const error = new Error('Non 2xx') @@ -384,7 +381,6 @@ class MultipartUploader { xhr.addEventListener('error', (ev) => { cleanup() - this.chunkState[index].busy = false const error = new Error('Unknown error') error.source = ev.target diff --git a/packages/@uppy/aws-s3-multipart/src/index.test.js b/packages/@uppy/aws-s3-multipart/src/index.test.js index 89aa67b7b48..fcd993071e0 100644 --- a/packages/@uppy/aws-s3-multipart/src/index.test.js +++ b/packages/@uppy/aws-s3-multipart/src/index.test.js @@ -54,13 +54,12 @@ describe('AwsS3Multipart', () => { }), completeMultipartUpload: jest.fn(async () => ({ location: 'test' })), abortMultipartUpload: jest.fn(), - prepareUploadParts: jest.fn(async () => { + prepareUploadParts: jest.fn(async (file, { parts }) => { const presignedUrls = {} - const possiblePartNumbers = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - possiblePartNumbers.forEach((partNumber) => { + parts.forEach(({ number }) => { presignedUrls[ - partNumber - ] = `https://bucket.s3.us-east-2.amazonaws.com/test/upload/multitest.dat?partNumber=${partNumber}&uploadId=6aeb1980f3fc7ce0b5454d25b71992&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIATEST%2F20210729%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20210729T014044Z&X-Amz-Expires=600&X-Amz-SignedHeaders=host&X-Amz-Signature=test` + number + ] = `https://bucket.s3.us-east-2.amazonaws.com/test/upload/multitest.dat?partNumber=${number}&uploadId=6aeb1980f3fc7ce0b5454d25b71992&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIATEST%2F20210729%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20210729T014044Z&X-Amz-Expires=600&X-Amz-SignedHeaders=host&X-Amz-Signature=test` }) return { presignedUrls } }), @@ -173,6 +172,81 @@ describe('AwsS3Multipart', () => { { ETag: 'test', PartNumber: 10 }, ]) }) + + it('Keeps chunks marked as busy through retries until they complete', async () => { + const scope = nock( + 'https://bucket.s3.us-east-2.amazonaws.com', + ).defaultReplyHeaders({ + 'access-control-allow-headers': '*', + 'access-control-allow-method': 'PUT', + 'access-control-allow-origin': '*', + 'access-control-expose-headers': 'ETag', + }) + + const fileSize = 50 * MB + + scope + .options((uri) => uri.includes('test/upload/multitest.dat')) + .reply(200, '') + scope + .put((uri) => uri.includes('test/upload/multitest.dat') && !uri.includes('partNumber=7')) + .reply(200, '', { ETag: 'test' }) + + // Fail the part 7 upload once, then let it succeed + let calls = 0 + scope + .put((uri) => uri.includes('test/upload/multitest.dat') && uri.includes('partNumber=7')) + .reply(() => (calls++ === 0 ? [500] : [200, '', { ETag: 'test' }])) + + scope.persist() + + // Spy on the busy/done state of the test chunk (part 7, chunk index 6) + let busySpy + let doneSpy + awsS3Multipart.setOptions({ + createMultipartUpload: jest.fn((file) => { + const multipartUploader = awsS3Multipart.uploaders[file.id] + const testChunkState = multipartUploader.chunkState[6] + let busy = false + let done = false + busySpy = jest.fn((value) => { busy = value }) + doneSpy = jest.fn((value) => { done = value }) + Object.defineProperty(testChunkState, 'busy', { get: () => busy, set: busySpy }) + Object.defineProperty(testChunkState, 'done', { get: () => done, set: doneSpy }) + + return { + uploadId: '6aeb1980f3fc7ce0b5454d25b71992', + key: 'test/upload/multitest.dat', + } + }), + }) + + core.addFile({ + source: 'jest', + name: 'multitest.dat', + type: 'application/octet-stream', + data: new File([new Uint8Array(fileSize)], { + type: 'application/octet-stream', + }), + }) + + await core.upload() + + // The chunk should be marked as done once + expect(doneSpy.mock.calls.length).toEqual(1) + expect(doneSpy.mock.calls[0][0]).toEqual(true) + + // Any changes that set busy to false should only happen after the chunk has been marked done, + // otherwise a race condition occurs (see PR #3955) + const doneCallOrderNumber = doneSpy.mock.invocationCallOrder[0] + for (const [index, callArgs] of busySpy.mock.calls.entries()) { + if (callArgs[0] === false) { + expect(busySpy.mock.invocationCallOrder[index]).toBeGreaterThan(doneCallOrderNumber) + } + } + + expect(awsS3Multipart.opts.prepareUploadParts.mock.calls.length).toEqual(3) + }) }) describe('MultipartUploader', () => {