From df5342d5c1d2a642c0aab8536f426208d8bb15e2 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 29 Apr 2022 11:13:11 -0700 Subject: [PATCH] fix(offline): Speed up offline storage by ~87% Closes #4166 --- lib/offline/storage.js | 85 +++++++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 34 deletions(-) diff --git a/lib/offline/storage.js b/lib/offline/storage.js index 1b6c0f7690..ff5e134b86 100644 --- a/lib/offline/storage.js +++ b/lib/offline/storage.js @@ -196,7 +196,7 @@ shaka.offline.Storage = class { goog.asserts.assert(typeof(config) == 'object', 'Should be an object!'); goog.asserts.assert( - this.config_, 'Cannot reconfigure stroage after calling destroy.'); + this.config_, 'Cannot reconfigure storage after calling destroy.'); return shaka.util.PlayerConfiguration.mergeConfigObjects( /* destination= */ this.config_, /* updates= */ config ); } @@ -441,36 +441,34 @@ shaka.offline.Storage = class { async downloadSegments_( toDownload, manifestId, manifestDB, downloader, config, storage, manifest, drmEngine) { + let pendingManifestUpdates = {}; + let pendingDataSize = 0; + /** * @param {!Array.} toDownload * @param {boolean} updateDRM */ const download = async (toDownload, updateDRM) => { - const throwIfAbortedFn = () => { - this.ensureNotDestroyed_(); - }; for (const download of toDownload) { - /** @param {?BufferSource} data */ - let data; const request = download.makeSegmentRequest(config); const estimateId = download.estimateId; const isInitSegment = download.isInitSegment; - const onDownloaded = (d) => { - data = d; - return Promise.resolve(); - }; - downloader.queue(download.groupId, - request, estimateId, isInitSegment, onDownloaded); - downloader.queueWork(download.groupId, async () => { - goog.asserts.assert(data, 'We should have loaded data by now'); - goog.asserts.assert(data instanceof ArrayBuffer, - 'The data should be an ArrayBuffer'); + const onDownloaded = async (data) => { + // Store the data. + const dataKeys = await storage.addSegments([{data}]); + this.ensureNotDestroyed_(); + + // Store the necessary update to the manifest, to be processed later. const ref = /** @type {!shaka.media.SegmentReference} */ ( download.ref); - manifestDB = (await shaka.offline.Storage.assignStreamToManifest( - manifestId, ref, {data}, throwIfAbortedFn)) || manifestDB; - }); + const id = shaka.offline.DownloadInfo.idForSegmentRef(ref); + pendingManifestUpdates[id] = dataKeys[0]; + pendingDataSize += data.byteLength; + }; + + downloader.queue(download.groupId, + request, estimateId, isInitSegment, onDownloaded); } await downloader.waitToFinish(); @@ -496,12 +494,30 @@ shaka.offline.Storage = class { await download(toDownload.filter((info) => info.isInitSegment), true); this.ensureNotDestroyed_(); toDownload = toDownload.filter((info) => !info.isInitSegment); + + // Copy these and reset them now, before calling await. + const manifestUpdates = pendingManifestUpdates; + const dataSize = pendingDataSize; + pendingManifestUpdates = {}; + pendingDataSize = 0; + + manifestDB = + (await shaka.offline.Storage.assignStreamToManifest( + manifestId, manifestUpdates, dataSize, + () => this.ensureNotDestroyed_())) || manifestDB; + this.ensureNotDestroyed_(); } if (!usingBgFetch) { await download(toDownload, false); this.ensureNotDestroyed_(); + manifestDB = + (await shaka.offline.Storage.assignStreamToManifest( + manifestId, pendingManifestUpdates, pendingDataSize, + () => this.ensureNotDestroyed_())) || manifestDB; + this.ensureNotDestroyed_(); + goog.asserts.assert( !manifestDB.isIncomplete, 'The manifest should be complete by now'); } else { @@ -538,19 +554,18 @@ shaka.offline.Storage = class { * changes of the other. * * @param {number} manifestId - * @param {!shaka.media.SegmentReference} ref - * @param {shaka.extern.SegmentDataDB} data + * @param {!Object.} manifestUpdates + * @param {number} dataSizeUpdate * @param {function()} throwIfAbortedFn A function that should throw if the * download has been aborted. * @return {!Promise.} */ - static async assignStreamToManifest(manifestId, ref, data, throwIfAbortedFn) { + static async assignStreamToManifest( + manifestId, manifestUpdates, dataSizeUpdate, throwIfAbortedFn) { /** @type {shaka.offline.StorageMuxer} */ const muxer = new shaka.offline.StorageMuxer(); - const idForRef = shaka.offline.DownloadInfo.idForSegmentRef(ref); let manifestUpdated = false; - let dataKey; let activeHandle; /** @type {!shaka.extern.ManifestDB} */ let manifestDB; @@ -561,11 +576,6 @@ shaka.offline.Storage = class { await muxer.init(); activeHandle = await muxer.getActive(); - // Store the data. - const dataKeys = await activeHandle.cell.addSegments([data]); - dataKey = dataKeys[0]; - throwIfAbortedFn(); - // Acquire the mutex before accessing the manifest, since there could be // multiple instances of this method running at once. mutexId = await shaka.offline.Storage.mutex_.acquire(); @@ -580,19 +590,25 @@ shaka.offline.Storage = class { let complete = true; for (const stream of manifestDB.streams) { for (const segment of stream.segments) { - if (segment.pendingSegmentRefId == idForRef) { + let dataKey = segment.pendingSegmentRefId ? + manifestUpdates[segment.pendingSegmentRefId] : null; + if (dataKey != null) { segment.dataKey = dataKey; // Now that the segment has been associated with the appropriate // dataKey, the pendingSegmentRefId is no longer necessary. segment.pendingSegmentRefId = undefined; } - if (segment.pendingInitSegmentRefId == idForRef) { + + dataKey = segment.pendingInitSegmentRefId ? + manifestUpdates[segment.pendingInitSegmentRefId] : null; + if (dataKey != null) { segment.initSegmentKey = dataKey; // Now that the init segment has been associated with the // appropriate initSegmentKey, the pendingInitSegmentRefId is no // longer necessary. segment.pendingInitSegmentRefId = undefined; } + if (segment.pendingSegmentRefId) { complete = false; } @@ -603,7 +619,7 @@ shaka.offline.Storage = class { } // Update the size of the manifest. - manifestDB.size += data.data.byteLength; + manifestDB.size += dataSizeUpdate; // Mark the manifest as complete, if all segments are downloaded. if (complete) { @@ -617,11 +633,12 @@ shaka.offline.Storage = class { } catch (e) { await shaka.offline.Storage.cleanStoredManifest(manifestId); - if (activeHandle && !manifestUpdated && dataKey) { + if (activeHandle && !manifestUpdated) { + const dataKeys = Object.values(manifestUpdates); // The cleanStoredManifest method will not "see" any segments that have // been downloaded but not assigned to the manifest yet. So un-store // them separately. - await activeHandle.cell.removeSegments([dataKey], (key) => {}); + await activeHandle.cell.removeSegments(dataKeys, (key) => {}); } throw e;