HLS image track / thumbnails: Some information is missing, timing is wrong in last segment

[dvoracek-slub]

Have you read the FAQ and checked for duplicate open issues?
Yes

What version of Shaka Player are you using?
3.2.0

Can you reproduce the issue with our latest release version?
Yes

Can you reproduce the issue with the latest code from master?
Yes

Are you using the demo app or your own custom app?
Custom

If custom app, can you reproduce the issue using our demo app?
No (it doesn't provide a thumbnail preview UI and I don't know how to access the player object)

What browser and OS are you using?
Linux Mint 20.1, Firefox 95.0.1

For embedded devices (smart TVs, etc.), what model and firmware version are you using?
–

What are the manifest and license server URIs?
https://raw.githubusercontent.com/dvoracek-slub/shaka-player-issue/main/hls/video.m3u8

(full example: https://raw.githubusercontent.com/dvoracek-slub/shaka-player-issue/main/hls/index.html)

What configuration are you using? What is the output of player.getConfiguration()?

{
  "drm": {
    "retryParameters": {
      "maxAttempts": 2,
      "baseDelay": 1000,
      "backoffFactor": 2,
      "fuzzFactor": 0.5,
      "timeout": 30000,
      "stallTimeout": 5000,
      "connectionTimeout": 10000
    },
    "servers": {},
    "clearKeys": {},
    "advanced": {},
    "delayLicenseRequestUntilPlayed": false,
    "logLicenseExchange": false,
    "updateExpirationTime": 1,
    "preferredKeySystems": []
  },
  "manifest": {
    "retryParameters": {
      "maxAttempts": 2,
      "baseDelay": 1000,
      "backoffFactor": 2,
      "fuzzFactor": 0.5,
      "timeout": 30000,
      "stallTimeout": 5000,
      "connectionTimeout": 10000
    },
    "availabilityWindowOverride": null,
    "disableAudio": false,
    "disableVideo": false,
    "disableText": false,
    "disableThumbnails": false,
    "defaultPresentationDelay": 0,
    "dash": {
      "clockSyncUri": "",
      "ignoreDrmInfo": false,
      "disableXlinkProcessing": false,
      "xlinkFailGracefully": false,
      "ignoreMinBufferTime": false,
      "autoCorrectDrift": true,
      "initialSegmentLimit": 1000,
      "ignoreSuggestedPresentationDelay": false,
      "ignoreEmptyAdaptationSet": false,
      "ignoreMaxSegmentDuration": false,
      "keySystemsByURI": {
        "urn:uuid:1077efec-c0b2-4d02-ace3-3c1e52e2fb4b": "org.w3.clearkey",
        "urn:uuid:edef8ba9-79d6-4ace-a3c8-27dcd51d21ed": "com.widevine.alpha",
        "urn:uuid:9a04f079-9840-4286-ab92-e65be0885f95": "com.microsoft.playready",
        "urn:uuid:79f0049a-4098-8642-ab92-e65be0885f95": "com.microsoft.playready",
        "urn:uuid:f239e769-efa3-4850-9c16-a903c6932efb": "com.adobe.primetime"
      }
    },
    "hls": {
      "ignoreTextStreamFailures": false,
      "ignoreImageStreamFailures": false,
      "useFullSegmentsForStartTime": false,
      "defaultAudioCodec": "mp4a.40.2",
      "defaultVideoCodec": "avc1.42E01E"
    }
  },
  "streaming": {
    "retryParameters": {
      "maxAttempts": 2,
      "baseDelay": 1000,
      "backoffFactor": 2,
      "fuzzFactor": 0.5,
      "timeout": 30000,
      "stallTimeout": 5000,
      "connectionTimeout": 10000
    },
    "rebufferingGoal": 2,
    "bufferingGoal": 10,
    "bufferBehind": 30,
    "ignoreTextStreamFailures": false,
    "alwaysStreamText": false,
    "startAtSegmentBoundary": false,
    "gapDetectionThreshold": 0.1,
    "smallGapLimit": 0.5,
    "jumpLargeGaps": false,
    "durationBackoff": 1,
    "forceTransmuxTS": false,
    "safeSeekOffset": 5,
    "stallEnabled": true,
    "stallThreshold": 1,
    "stallSkip": 0.1,
    "useNativeHlsOnSafari": true,
    "inaccurateManifestTolerance": 2,
    "lowLatencyMode": false,
    "autoLowLatencyMode": false,
    "forceHTTPS": false,
    "preferNativeHls": false,
    "updateIntervalSeconds": 1,
    "dispatchAllEmsgBoxes": false,
    "observeQualityChanges": false
  },
  "offline": {
    "usePersistentLicense": true
  },
  "abr": {
    "enabled": true,
    "useNetworkInformation": true,
    "defaultBandwidthEstimate": 1000000,
    "switchInterval": 8,
    "bandwidthUpgradeTarget": 0.85,
    "bandwidthDowngradeTarget": 0.95,
    "restrictions": {
      "minWidth": 0,
      "maxWidth": null,
      "minHeight": 0,
      "maxHeight": null,
      "minPixels": 0,
      "maxPixels": null,
      "minFrameRate": 0,
      "maxFrameRate": null,
      "minBandwidth": 0,
      "maxBandwidth": null
    },
    "advanced": {
      "minTotalBytes": 128000,
      "minBytes": 16000,
      "fastHalfLife": 2,
      "slowHalfLife": 5
    }
  },
  "preferredAudioLanguage": "",
  "preferredTextLanguage": "",
  "preferredVariantRole": "",
  "preferredTextRole": "",
  "preferredAudioChannelCount": 2,
  "preferredVideoCodecs": [],
  "preferredAudioCodecs": [],
  "preferForcedSubs": false,
  "preferredDecodingAttributes": [],
  "restrictions": {
    "minWidth": 0,
    "maxWidth": null,
    "minHeight": 0,
    "maxHeight": null,
    "minPixels": 0,
    "maxPixels": null,
    "minFrameRate": 0,
    "maxFrameRate": null,
    "minBandwidth": 0,
    "maxBandwidth": null
  },
  "playRangeStart": 0,
  "playRangeEnd": null,
  "cmcd": {
    "enabled": false,
    "sessionId": "",
    "contentId": "",
    "useHeaders": false
  }
}

What did you do? What did you expect to happen? What actually happened?

In our video player, I would like to present a thumbnail preview based on the (tiled) image tracks referenced in the HLS playlist.

const imgTrack = player.getImageTracks()[0];
const thumb = await player.getThumbnails(imgTrack.id, /* time= */ 4);

// (1a) Expected: 80, 45
//      Actual: null, null
console.log(imgTrack.width, imgTrack.height);

// (1b) Expected: 2880
//      Actual: 0
console.log(imgTrack.bandwidth);

// (2) Expected: 80, 0
//     Actual: 0, 0
console.log(thumb.positionX, thumb.positionY);

// (3a) Expected: 4
//      Actual: 3.72333
console.log(thumb.startTime);

// (3b) Expected: 4
//      Actual: 3.72333
console.log(thumb.duration);

Notes
Apparently, image size and bandwidth aren't processed for EXT-X-IMAGE-STREAM-INF in the master playlist. I would like to use this information to select among thumbnail sets (in particular, to load low-resolution thumbnails first and then switch to a higher-resolution thumbnail).

Consequently positionX == 0, because getThumbnails() now assumes a width and height of zero. (However, each segment in the image playlist may also specify its own resolution, so it may be good to use that...)

As for the times, because the EXTINF of the image segment is longer than the video, it seems that the player somehow abridges the segment's length and uses that to calculate thumbnail timings.

[theodab]

It looks like we're not reading the DURATION attribute on EXT-X-TILES (which is where the duration of 3.72333 comes from; that's a 373 second playlist divided by 100 thumbnails), or the RESOLUTION and BANDWIDTH attributes on EXT-X-IMAGE-STREAM-INF.

[dvoracek-slub]

Thanks for fixing the issue so quickly!

Now for the DASH version, I get an error when calling getImageTracks().

const video = document.querySelector("video");
const player = new shaka.Player(video);

await player.load(
  "https://raw.githubusercontent.com/dvoracek-slub/shaka-player-issue/main/hls/video.mpd"
);

player.getImageTracks();

Output:

Uncaught (in promise) TypeError: a.segmentIndex is null
    imageStreamToTrack http://127.0.0.1:8080/shaka-player.ui.debug.js:378
    getImageTracks http://127.0.0.1:8080/shaka-player.ui.debug.js:1044
    getImageTracks http://127.0.0.1:8080/shaka-player.ui.debug.js:1044
    <anonymous> http://127.0.0.1:8080/:21
    async* http://127.0.0.1:8080/:51
shaka-player.ui.debug.js:378:139

[avelad]

@shaka-bot reopen

[avelad]

The commit 264c842 introduce a regression in DASH!

The stream.segmentIndex.get(0) is undefined in DASH.
https://github.com/google/shaka-player/blob/master/lib/util/stream_utils.js#L997

@joeyparrish can you review it?

[theodab]

Ah, whoops, I forgot that the segmentIndex would be null for DASH manifests. It should be a simple fix, luckily.