Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: switch to target latency for live sync and add dynamic target latency feature #6193

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
Open

feat: switch to target latency for live sync and add dynamic target latency feature #6193

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
f59ac9b
parse target latency
gkatsev Jan 29, 2024
c2460fa
add dynamic target latency options
gkatsev Jan 29, 2024
b7b692c
switch to use target latency
gkatsev Jan 30, 2024
4fce540
dynamic target latency
gkatsev Jan 30, 2024
f7af735
fixup behavior
gkatsev Jan 31, 2024
e4b2818
add target latency and dynamic target latency options to demo page
gkatsev Jan 31, 2024
96d5b79
make some demo items as "can be zero" so that error wont show up for …
gkatsev Jan 31, 2024
7826d4b
fixup var rebase
gkatsev Feb 2, 2024
60c1433
reset target latency reached in unload
gkatsev Feb 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
21 changes: 19 additions & 2 deletions demo/config.js
View file
Open in desktop
Expand Up @@ -462,24 +462,41 @@ shakaDemo.Config = class {
.addBoolInput_('Disable Video Prefetch',
'streaming.disableVideoPrefetch')
.addBoolInput_('Live Sync', 'streaming.liveSync')
.addNumberInput_('Target latency for live sync',
'streaming.liveSyncTargetLatency',
/* canBeDecimal= */ true,
/* canBeZero= */ true)
.addNumberInput_('Target latency tolerance',
'streaming.liveSyncTargetLatencyTolerance',
/* canBeDecimal= */ true,
/* canBeZero= */ true)
.addNumberInput_('Max latency for live sync',
'streaming.liveSyncMaxLatency',
/* canBeDecimal= */ true,
/* canBeZero= */ true)
.addNumberInput_('Playback rate for live sync',
'streaming.liveSyncPlaybackRate',
/* canBeDecimal= */ true,
/* canBeZero= */ false)
/* canBeZero= */ true)
.addNumberInput_('Min latency for live sync',
'streaming.liveSyncMinLatency',
/* canBeDecimal= */ true,
/* canBeZero= */ true)
.addNumberInput_('Min playback rate for live sync',
'streaming.liveSyncMinPlaybackRate',
/* canBeDecimal= */ true)
/* canBeDecimal= */ true,
/* canBeZero= */ true)
.addBoolInput_('Live Sync Panic Mode', 'streaming.liveSyncPanicMode')
.addNumberInput_('Live Sync Panic Mode Threshold',
'streaming.liveSyncPanicThreshold')
.addBoolInput_('Dynamic Target Latency',
'streaming.liveSyncDynamicTargetLatency')
.addNumberInput_('Dynamic Target Latency Stability Threshold',
'streaming.liveSyncDynamicTargetLatencyStabilityThreshold')
.addNumberInput_('Dynamic Target Latency Rebuffer Increment',
'streaming.liveSyncDynamicTargetLatencyRebufferIncrement',
/* canBeDecimal= */ true,
/* canBeZero= */ true)
.addBoolInput_('Allow Media Source recoveries',
'streaming.allowMediaSourceRecoveries')
.addNumberInput_('Minimum time between recoveries',
Expand Down
3 changes: 3 additions & 0 deletions externs/shaka/manifest.js
View file
Open in desktop
Expand Up @@ -151,6 +151,7 @@ shaka.extern.InitDataOverride;

/**
* @typedef {{
* targetLatency:?number,
* maxLatency: ?number,
* maxPlaybackRate: ?number,
* minLatency: ?number,
Expand All @@ -164,6 +165,8 @@ shaka.extern.InitDataOverride;
* minPlaybackRate to increase latency.
* More information {@link https://dashif.org/docs/CR-Low-Latency-Live-r8.pdf here}.
*
* @property {?number} targetLatency
* The target latency to aim for.
* @property {?number} maxLatency
* Maximum latency in seconds.
* @property {?number} maxPlaybackRate
Expand Down
28 changes: 28 additions & 0 deletions externs/shaka/player.js
View file
Open in desktop
Expand Up @@ -1164,12 +1164,17 @@ shaka.extern.ManifestConfiguration;
* disableTextPrefetch: boolean,
* disableVideoPrefetch: boolean,
* liveSync: boolean,
* liveSyncTargetLatency: number,
* liveSyncTargetLatencyTolerance: number,
* liveSyncMaxLatency: number,
* liveSyncPlaybackRate: number,
* liveSyncMinLatency: number,
* liveSyncMinPlaybackRate: number,
* liveSyncPanicMode: boolean,
* liveSyncPanicThreshold: number,
* liveSyncDynamicTargetLatency: boolean,
* liveSyncDynamicTargetLatencyStabilityThreshold: number,
* liveSyncDynamicTargetLatencyRebufferIncrement: number,
* allowMediaSourceRecoveries: boolean,
* minTimeBetweenRecoveries: number,
* vodDynamicPlaybackRate: boolean,
Expand Down Expand Up @@ -1312,6 +1317,12 @@ shaka.extern.ManifestConfiguration;
* rate. Defaults to <code>false</code>.
* Note: on some SmartTVs, if this is activated, it may not work or the sound
* may be lost when activated.
* @property {number} liveSyncTargetLatency
* Preferred latency, in seconds. Effective only if liveSync is true.
* Defaults to <code>0.5</code>.
* @property {number} liveSyncTargetLatencyTolerance
* Latency tolerance for target latency, in seconds. Effective only if
* liveSync is true. Defaults to <code>0.5</code>.
* @property {number} liveSyncMaxLatency
* Maximum acceptable latency, in seconds. Effective only if liveSync is
* true. Defaults to <code>1</code>.
Expand All @@ -1334,6 +1345,23 @@ shaka.extern.ManifestConfiguration;
* @property {number} liveSyncPanicThreshold
* Number of seconds that playback stays in panic mode after a rebuffering.
* Defaults to <code>60</code>
* @property {boolean} liveSyncDynamicTargetLatency
* If <code>true</code>, dynamic latency for live sync is enabled. When
* enabled, the target latency will be adjusted closer to the min latency
* when playback is stable
* (see <code>liveSyncDynamicTargetLatencyStabilityThreshold</code>). If
* there are rebuffering events, then the target latency will move towards
* the max latency value in increments of
* <code>liveSyncDynamicTargetLatencyRebufferIncrement</code>. Defaults to
* <code>false</code>.
* @property {number} liveSyncDynamicTargetLatencyRebufferIncrement
* The value, in seconds, to increment the target latency towards
* <code>liveSyncMaxLatency</code> after a rebuffering event. Defaults to
* <code>0.5</code>.
* @property {number} liveSyncDynamicTargetLatencyStabilityThreshold
* Number of seconds after a rebuffering before we are considered stable and
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems to me that we are setting up a pendulum. Am I missing something in this analysis?

When we aim for a min latency of 0s (default), if that is not achievable, we could rebuffer. Then the latency will be backed off in 0.5s increments (default) until we find stable playback. If we sustain stable playback at a particular latency target for 60s (default), we will reduce the target again (increment not documented... same as for increases?) until we hit an unsustainable level and rebuffer.

It seems like this pendulum swings between the best latency target we can achieve, and a rebuffer event, back and forth. This would seem to set the user up for continued rebuffering events.

Am I missing something? I know I've missed a lot of changes in the time I've been away.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you have a good overview of the changes. I've also previousl written a description of this here #6131 (comment).

For increment to go towards min latency is half of the distance between target latency and min latency.
Additonally, it isn't just backing off in 0.5s increments, but rather in a multiple of 0.5s per each rebuffering event. So, if there's a bunch of rebuffering, we'll increment further back. https://github.com/shaka-project/shaka-player/pull/6193/files#diff-7abf4aa2639707ce10a19703663841fae6c5638809faf4dfb2f1894791f154ddR5694.

I think that the pendulum that you mention is definitely a possibility, though, I assume that folks that may be using this option will probably want to tweak their target and minimum settings away from the defaults. It probably doesn't make much difference with the default min, max, and target latency values.
Also, I'm not sure that a pendulum here is necessarily an issue. During playback we do want to react to occasions when there are rebuffering, so, back off on being close to the live edge and if playback is stable, we can move closer to the edge. Given the unpredicatiblity of the network, I think it's reasonable, particularly for an opt-in feature. Some internal tests, a feature like this decreased the live latency and reduced the rebuffering count.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe if the rebuffer count is high we should stick to the max latency and stop trying to move it towards min?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure that a pendulum here is necessarily an issue. During playback we do want to react to occasions when there are rebuffering, so, back off on being close to the live edge and if playback is stable, we can move closer to the edge. Given the unpredicatiblity of the network, I think it's reasonable, particularly for an opt-in feature. Some internal tests, a feature like this decreased the live latency and reduced the rebuffering count.

I hear you. But I wonder what the behavior of this will be further from the median experience. We may not know that until it is deployed and instrumented across a large number of sessions. Is your "internal testing" based on real user populations in an A/B experiment? Or just "seems good" from your fast connection at work or home? I mean no offense in this. I'm always forced to rely on the latter myself, but I just think we should be clear about the limitations of it when we discuss our risk/reward assessments.

I assume that folks that may be using this option will probably want to tweak their target and minimum settings away from the defaults. It probably doesn't make much difference with the default min, max, and target latency values.

While I'm sure plenty of developers are power users who understand the nuances and can tweak all these things, I bet there are still many who don't have a deep enough knowledge or experience to do so. Or who would Deploy an updated player without noticing that this feature is in it. Are there better defaults we could pick? I feel like we should assume that even if it's opt-in right now, it could become opt-out some day, and we should strive for defaults that are a good starting point for most scenarios, with low risk if possible.

maybe if the rebuffer count is high we should stick to the max latency and stop trying to move it towards min?

Maybe so. Maybe something like this, though I'm not certain what. I feel strongly that something should exist to stop a rebuffering cycle if it occurs.

I feel the same about latency as resolution: better to be a little father from the bleeding edge if you can avoid buffering in most cases.

But reasonable people could certainly disagree with me on that. I also don't have any user session data to back up my philosophies on any of this. So I would defer to real data if anyone has some.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I totally see your concerns as well. Definitely a tricky situation trying to cater to power users while not letting others accidentally ruin their user experience.

I believe the tests were done with a network emulator and a bunch of network traces, but getting confirmation. Either way, it's definitely not the same as a full A/B experiment, though, definitely better than someone just hitting play on their laptop a bunch of times.

Would liveSyncDynamicTargetLatencyRebufferThreshold, where once the rebuffering count cross this boundary, it will choose the max latency and stick there without trying to push closer to the edge again assuage your concerns?

Also, given that this change is essentially two in one: switch to target latency, and dynamic target latency. With shaka v5 looming, if we don't have agreement on the dynamic part, would just the target latency be able to land? Since dynamic target latency could later be added as a feature, if it doesn't happen before v5 is done. Happy to separate this PR into two if that makes it more likely for the breaking change to land in time for v5.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Confirmed my interpretation was correct regarding the test runner.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What would be the next action?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oops, somehow missed your message.

For next actions, there are several options. The one that would be the least amount of work is likely to add the liveSyncDynamicTargetLatencyRebufferThreshold or similar option. Alternatively, I can split this PR between the refactor to target latency and the new feature of target latency. This was more necessary when v5 was scheduled soon.

I'd like us to have consensus on the next course of action before I update the PR. Is liveSyncDynamicTargetLatencyRebufferThreshold reasonable? Perhaps with a new name. Or maybe there are other ideas for resolving @joeyparrish's concern about the pendulum. Currently, I don't have any other ideas, but I'll spend some more time thinking about it next week. I would appreciate any other suggestions.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another potential change is to move the dynamic target latency options into a sub config object.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I think it's better to split it into two PRs, it would be easier to review. As for the new setup, I'll let Joey decide.

* will move the target latency towards <code>liveSyncMinLatency</code>
* value. Defaults to <code>60</code>
* @property {boolean} allowMediaSourceRecoveries
* Indicate if we should recover from VIDEO_ERROR resetting Media Source.
* Defaults to <code>true</code>.
Expand Down
50 changes: 30 additions & 20 deletions lib/dash/dash_parser.js
View file
Open in desktop
Expand Up @@ -723,29 +723,39 @@ shaka.dash.DashParser = class {
const latencyNode = TXml.findChild(elem, 'Latency');
const playbackRateNode = TXml.findChild(elem, 'PlaybackRate');

if ((latencyNode && latencyNode.attributes['max']) || playbackRateNode) {
const maxLatency = latencyNode && latencyNode.attributes['max'] ?
parseInt(latencyNode.attributes['max'], 10) / 1000 :
null;
const maxPlaybackRate = playbackRateNode ?
parseFloat(playbackRateNode.attributes['max']) :
null;
const minLatency = latencyNode && latencyNode.attributes['min'] ?
parseInt(latencyNode.attributes['min'], 10) / 1000 :
null;
const minPlaybackRate = playbackRateNode ?
parseFloat(playbackRateNode.attributes['min']) :
null;
if (!latencyNode && !playbackRateNode) {
return null;
}

return {
maxLatency,
maxPlaybackRate,
minLatency,
minPlaybackRate,
};
const description = {};

if (latencyNode) {
if ('target' in latencyNode.attributes) {
description.targetLatency =
parseInt(latencyNode.attributes['target'], 10) / 1000;
}
if ('max' in latencyNode.attributes) {
description.maxLatency =
parseInt(latencyNode.attributes['max'], 10) / 1000;
}
if ('min' in latencyNode.attributes) {
description.minLatency =
parseInt(latencyNode.attributes['min'], 10) / 1000;
}
}

return null;
if (playbackRateNode) {
if ('max' in playbackRateNode.attributes) {
description.maxPlaybackRate =
parseFloat(playbackRateNode.attributes['max']);
}
if ('min' in playbackRateNode.attributes) {
description.minPlaybackRate =
parseFloat(playbackRateNode.attributes['min']);
}
}

return description;
}

/**
Expand Down
95 changes: 79 additions & 16 deletions lib/player.js
View file
Open in desktop
Expand Up @@ -672,6 +672,15 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
/** @private {?shaka.extern.PlayerConfiguration} */
this.config_ = this.defaultConfig_();

/** @private {?number} */
this.currentTargetLatency_ = null;

/** @private {number} */
this.rebufferingCount_ = -1;

/** @private {?number} */
this.targetLatencyReached_ = null;
avelad marked this conversation as resolved.
Show resolved Hide resolved

/**
* The TextDisplayerFactory that was last used to make a text displayer.
* Stored so that we can tell if a new type of text displayer is desired.
Expand Down Expand Up @@ -5657,6 +5666,34 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
this.cmcdManager_.setBuffering(isBuffering);
}
this.updateStateHistory_();

const dynamicTargetLatency =
this.config_.streaming.liveSyncDynamicTargetLatency;
if (dynamicTargetLatency && isBuffering) {
let maxLatency;
if (this.config_.streaming.liveSync) {
maxLatency = this.config_.streaming.liveSyncMaxLatency;
} else if (this.manifest_ && this.manifest_.serviceDescription) {
maxLatency = this.manifest_.serviceDescription.maxLatency ||
this.config_.streaming.liveSyncMaxLatency;
} else {
// default maxLatency to a high number since we will be doing a
// Math.min below so the target value will be chosen in case a max
// latency wasn't provided
maxLatency = Number.MAX_SAFE_INTEGER;
}

const targetLatencyTolerance =
this.config_.streaming.liveSyncTargetLatencyTolerance;
const rebufferIncrement =
this.config_.streaming.liveSyncDynamicTargetLatencyRebufferIncrement;
if (this.currentTargetLatency_) {
this.currentTargetLatency_ = Math.min(
this.currentTargetLatency_ +
++this.rebufferingCount_ * rebufferIncrement,
maxLatency - targetLatencyTolerance);
}
}
}

// Surface the buffering event so that the app knows if/when we are
Expand Down Expand Up @@ -5783,32 +5820,35 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
return;
}

let liveSyncTargetLatency;
let liveSyncMaxLatency;
let liveSyncPlaybackRate;
let liveSyncMinLatency;
let liveSyncMinPlaybackRate;
const liveSyncTargetLatencyTolerance =
this.config_.streaming.liveSyncTargetLatencyTolerance;
const dynamicTargetLatency =
this.config_.streaming.liveSyncDynamicTargetLatency;
const stabilityThreshold =
this.config_.streaming.liveSyncDynamicTargetLatencyStabilityThreshold;
if (this.config_.streaming.liveSync) {
liveSyncTargetLatency = this.config_.streaming.liveSyncTargetLatency;
liveSyncMaxLatency = this.config_.streaming.liveSyncMaxLatency;
liveSyncPlaybackRate = this.config_.streaming.liveSyncPlaybackRate;
liveSyncMinLatency = this.config_.streaming.liveSyncMinLatency;
liveSyncMinPlaybackRate = this.config_.streaming.liveSyncMinPlaybackRate;
} else {
// serviceDescription must override if it is defined in the MPD and
// liveSync configuration is not set.
if (this.manifest_ && this.manifest_.serviceDescription) {
liveSyncTargetLatency =
this.manifest_.serviceDescription.targetLatency ||
this.config_.streaming.liveSyncTargetLatency;
liveSyncMaxLatency = this.manifest_.serviceDescription.maxLatency ||
this.config_.streaming.liveSyncMaxLatency;
liveSyncPlaybackRate =
this.manifest_.serviceDescription.maxPlaybackRate ||
this.config_.streaming.liveSyncPlaybackRate;
}
}

let liveSyncMinLatency;
let liveSyncMinPlaybackRate;
if (this.config_.streaming.liveSync) {
liveSyncMinLatency = this.config_.streaming.liveSyncMinLatency;
liveSyncMinPlaybackRate = this.config_.streaming.liveSyncMinPlaybackRate;
} else {
// serviceDescription must override if it is defined in the MPD and
// liveSync configuration is not set.
if (this.manifest_ && this.manifest_.serviceDescription) {
liveSyncMinLatency = this.manifest_.serviceDescription.minLatency ||
this.config_.streaming.liveSyncMinLatency;
liveSyncMinPlaybackRate =
Expand All @@ -5817,6 +5857,23 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
}
}

if (!this.currentTargetLatency_ && liveSyncTargetLatency) {
this.currentTargetLatency_ = liveSyncTargetLatency;
}

if (dynamicTargetLatency && this.currentTargetLatency_ &&
liveSyncTargetLatency && typeof liveSyncMinLatency == 'number' &&
this.targetLatencyReached_ &&
(Date.now() - this.targetLatencyReached_) > stabilityThreshold * 1000) {
const latencyIncrement = (liveSyncTargetLatency - liveSyncMinLatency) / 2;
this.currentTargetLatency_ = Math.max(
this.currentTargetLatency_ - latencyIncrement,
// current target latency should be within the tolerance of the min
// latency to not overshoot it
liveSyncMinLatency + liveSyncTargetLatencyTolerance);
this.targetLatencyReached_ = null;
}

const latency = seekRange.end - this.video_.currentTime;
let offset = 0;
// In src= mode, the seek range isn't updated frequently enough, so we need
Expand All @@ -5830,6 +5887,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
offset = Math.max(liveSyncPlaybackRate, bufferedEnd - seekRange.end);
}
}
const adjustedLatency = latency - offset;

const panicMode = this.config_.streaming.liveSyncPanicMode;
const panicThreshold = this.config_.streaming.liveSyncPanicThreshold * 1000;
Expand All @@ -5848,24 +5906,29 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
's). Updating playbackRate to ' + liveSyncMinPlaybackRate);
this.trickPlay(liveSyncMinPlaybackRate);
}
} else if (liveSyncMaxLatency && liveSyncPlaybackRate &&
(latency - offset) > liveSyncMaxLatency) {
} else if (this.currentTargetLatency_ && liveSyncPlaybackRate &&
(adjustedLatency >
this.currentTargetLatency_ + liveSyncTargetLatencyTolerance)) {
if (playbackRate != liveSyncPlaybackRate) {
shaka.log.debug('Latency (' + latency + 's) ' +
'is greater than liveSyncMaxLatency (' + liveSyncMaxLatency + 's). ' +
'Updating playbackRate to ' + liveSyncPlaybackRate);
this.trickPlay(liveSyncPlaybackRate);
}
} else if (liveSyncMinLatency && liveSyncMinPlaybackRate &&
(latency - offset) < liveSyncMinLatency) {
this.targetLatencyReached_ = null;
} else if (this.currentTargetLatency_ && liveSyncMinPlaybackRate &&
(adjustedLatency <
this.currentTargetLatency_ - liveSyncTargetLatencyTolerance)) {
if (playbackRate != liveSyncMinPlaybackRate) {
shaka.log.debug('Latency (' + latency + 's) ' +
'is smaller than liveSyncMinLatency (' + liveSyncMinLatency + 's). ' +
'Updating playbackRate to ' + liveSyncMinPlaybackRate);
this.trickPlay(liveSyncMinPlaybackRate);
}
this.targetLatencyReached_ = null;
} else if (playbackRate !== this.playRateController_.getDefaultRate()) {
this.cancelTrickPlay();
this.targetLatencyReached_ = Date.now();
}
}

Expand Down
5 changes: 5 additions & 0 deletions lib/util/player_configuration.js
View file
Open in desktop
Expand Up @@ -227,12 +227,17 @@ shaka.util.PlayerConfiguration = class {
disableTextPrefetch: false,
disableVideoPrefetch: false,
liveSync: false,
liveSyncTargetLatency: 1,
liveSyncTargetLatencyTolerance: 0.5,
liveSyncMaxLatency: 1,
liveSyncPlaybackRate: 1.1,
liveSyncMinLatency: 0,
liveSyncMinPlaybackRate: 1,
liveSyncPanicMode: false,
liveSyncPanicThreshold: 60,
liveSyncDynamicTargetLatency: false,
liveSyncDynamicTargetLatencyRebufferIncrement: 0.5,
liveSyncDynamicTargetLatencyStabilityThreshold: 60,
allowMediaSourceRecoveries: true,
minTimeBetweenRecoveries: 5,
vodDynamicPlaybackRate: false,
Expand Down