diff --git a/RELEASENOTES.md b/RELEASENOTES.md
index 0ccde8e702..4b08366d42 100644
--- a/RELEASENOTES.md
+++ b/RELEASENOTES.md
@@ -3,6 +3,7 @@
 ### Unreleased changes
 
 *   Common Library:
+    *   Add `Format.labels` to allow localized or other alternatives labels.
 *   ExoPlayer:
     *   Fix issue where `PreloadMediaPeriod` cannot retain the streams when it
         is preloaded again.
@@ -100,6 +101,8 @@
 *   RTMP Extension:
 *   HLS Extension:
 *   DASH Extension:
+    *   Populate all `Label` elements from the manifest into `Format.labels`
+        ([#1054](https://github.com/androidx/media/pull/1054)).
 *   Smooth Streaming Extension:
 *   RTSP Extension:
     *   Skip empty session information values (i-tags) in SDP parsing
diff --git a/libraries/common/src/main/java/androidx/media3/common/Format.java b/libraries/common/src/main/java/androidx/media3/common/Format.java
index 28e19abed5..cda7ac23e9 100644
--- a/libraries/common/src/main/java/androidx/media3/common/Format.java
+++ b/libraries/common/src/main/java/androidx/media3/common/Format.java
@@ -15,6 +15,7 @@
  */
 package androidx.media3.common;
 
+import static androidx.media3.common.util.Assertions.checkState;
 import static java.lang.annotation.ElementType.TYPE_USE;
 
 import android.os.Bundle;
@@ -140,7 +141,7 @@ public static final class Builder {
 
     @Nullable private String id;
     @Nullable private String label;
-    @Nullable private List<Label> labels;
+    private List<Label> labels;
     @Nullable private String language;
     private @C.SelectionFlags int selectionFlags;
     private @C.RoleFlags int roleFlags;
@@ -196,6 +197,7 @@ public static final class Builder {
 
     /** Creates a new instance with default values. */
     public Builder() {
+      labels = ImmutableList.of();
       averageBitrate = NO_VALUE;
       peakBitrate = NO_VALUE;
       // Sample specific.
@@ -298,6 +300,9 @@ public Builder setId(int id) {
     /**
      * Sets {@link Format#label}. The default value is {@code null}.
      *
+     * <p>If both this default label and a list of {@link #setLabels labels} are set, this default
+     * label must be part of label list.
+     *
      * @param label The {@link Format#label}.
      * @return The builder.
      */
@@ -308,14 +313,17 @@ public Builder setLabel(@Nullable String label) {
     }
 
     /**
-     * Sets {@link Format#labels}. The default value is {@code null}.
+     * Sets {@link Format#labels}. The default value is an empty list.
+     *
+     * <p>If both the default {@linkplain #setLabel label} and this list are set, the default label
+     * must be part of this list of labels.
      *
      * @param labels The {@link Format#labels}.
      * @return The builder.
      */
     @CanIgnoreReturnValue
-    public Builder setLabels(@Nullable List<Label> labels) {
-      this.labels = labels;
+    public Builder setLabels(List<Label> labels) {
+      this.labels = ImmutableList.copyOf(labels);
       return this;
     }
 
@@ -757,10 +765,20 @@ public Format build() {
   /** An identifier for the format, or null if unknown or not applicable. */
   @Nullable public final String id;
 
-  /** The human readable label, or null if unknown or not applicable. */
+  /**
+   * The default human readable label, or null if unknown or not applicable.
+   *
+   * <p>If non-null, the same label will be part of {@link #labels} too. If null, {@link #labels}
+   * will be empty.
+   */
   @Nullable public final String label;
 
-  /** The human readable list of labels, or null if unknown or not applicable. */
+  /**
+   * The human readable list of labels, or an empty list if unknown or not applicable.
+   *
+   * <p>If non-empty, the default {@link #label} will be part of this list. If empty, the default
+   * {@link #label} will be null.
+   */
   @UnstableApi public final List<Label> labels;
 
   /** The language as an IETF BCP 47 conformant tag, or null if unknown or not applicable. */
@@ -952,17 +970,19 @@ public Format build() {
   private Format(Builder builder) {
     id = builder.id;
     language = Util.normalizeLanguageCode(builder.language);
-    @Nullable String tmpLabel = builder.label;
-    labels = builder.labels == null ? new ArrayList<>() : builder.labels;
-    if (labels.isEmpty() && !TextUtils.isEmpty(tmpLabel)) {
-      labels.add(new Label(language, tmpLabel));
-      label = tmpLabel;
-    } else if (!labels.isEmpty() && TextUtils.isEmpty(tmpLabel)) {
-      label = getDefaultLabel(tmpLabel, labels);
+    if (builder.labels.isEmpty() && builder.label != null) {
+      labels = ImmutableList.of(new Label(language, builder.label));
+      label = builder.label;
+    } else if (!builder.labels.isEmpty() && builder.label == null) {
+      labels = builder.labels;
+      label = getDefaultLabel(builder.labels, language);
     } else {
-      label = tmpLabel;
+      checkState(
+          (builder.labels.isEmpty() && builder.label == null)
+              || (builder.labels.stream().anyMatch(l -> l.value.equals(builder.label))));
+      labels = builder.labels;
+      label = builder.label;
     }
-    checkLabels(label, labels);
     selectionFlags = builder.selectionFlags;
     roleFlags = builder.roleFlags;
     averageBitrate = builder.averageBitrate;
@@ -1010,29 +1030,6 @@ private Format(Builder builder) {
     }
   }
 
-  private @Nullable String getDefaultLabel(@Nullable String label, List<Label> labels) {
-    if (TextUtils.isEmpty(label)) {
-      for (Label l : labels) {
-        if (TextUtils.equals(l.lang, language)) {
-          return l.value;
-        }
-      }
-      if (!labels.isEmpty()) {
-        return labels.get(0).value;
-      }
-    }
-    return label;
-  }
-
-  private void checkLabels(@Nullable String label, List<Label> labels)
-      throws IllegalStateException {
-    if (!TextUtils.isEmpty(label)
-        && !labels.isEmpty()
-        && labels.stream().noneMatch(l -> TextUtils.equals(l.value, label))) {
-      throw new IllegalStateException();
-    }
-  }
-
   /** Returns a {@link Format.Builder} initialized with the values of this instance. */
   @UnstableApi
   public Builder buildUpon() {
@@ -1056,7 +1053,7 @@ public Format withManifestFormatInfo(Format manifestFormat) {
 
     // Prefer manifest values, but fill in from sample format if missing.
     @Nullable String label = manifestFormat.label != null ? manifestFormat.label : this.label;
-    List<Label> labels = manifestFormat.labels;
+    List<Label> labels = !manifestFormat.labels.isEmpty() ? manifestFormat.labels : this.labels;
     @Nullable String language = this.language;
     if ((trackType == C.TRACK_TYPE_TEXT || trackType == C.TRACK_TYPE_AUDIO)
         && manifestFormat.language != null) {
@@ -1246,6 +1243,7 @@ public boolean equals(@Nullable Object obj) {
         && Float.compare(pixelWidthHeightRatio, other.pixelWidthHeightRatio) == 0
         && Util.areEqual(id, other.id)
         && Util.areEqual(label, other.label)
+        && labels.equals(other.labels)
         && Util.areEqual(codecs, other.codecs)
         && Util.areEqual(containerMimeType, other.containerMimeType)
         && Util.areEqual(sampleMimeType, other.sampleMimeType)
@@ -1254,8 +1252,7 @@ public boolean equals(@Nullable Object obj) {
         && Util.areEqual(metadata, other.metadata)
         && Util.areEqual(colorInfo, other.colorInfo)
         && Util.areEqual(drmInitData, other.drmInitData)
-        && initializationDataEquals(other)
-        && labels.equals(other.labels);
+        && initializationDataEquals(other);
   }
 
   /**
@@ -1338,7 +1335,7 @@ public static String toLogString(@Nullable Format format) {
     if (format.language != null) {
       builder.append(", language=").append(format.language);
     }
-    if (format.labels.size() > 0) {
+    if (!format.labels.isEmpty()) {
       builder.append(", labels=[");
       Joiner.on(',').appendTo(builder, format.labels);
       builder.append("]");
@@ -1561,4 +1558,13 @@ private static String keyForInitializationData(int initialisationDataIndex) {
   private static <T> T defaultIfNull(@Nullable T value, @Nullable T defaultValue) {
     return value != null ? value : defaultValue;
   }
+
+  private static String getDefaultLabel(List<Label> labels, @Nullable String language) {
+    for (Label l : labels) {
+      if (TextUtils.equals(l.language, language)) {
+        return l.value;
+      }
+    }
+    return labels.get(0).value;
+  }
 }
diff --git a/libraries/common/src/main/java/androidx/media3/common/Label.java b/libraries/common/src/main/java/androidx/media3/common/Label.java
index 63e6c94501..455ed70dc0 100644
--- a/libraries/common/src/main/java/androidx/media3/common/Label.java
+++ b/libraries/common/src/main/java/androidx/media3/common/Label.java
@@ -1,94 +1,86 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package androidx.media3.common;
 
 import static androidx.media3.common.util.Assertions.checkNotNull;
 
 import android.os.Bundle;
-import android.os.Parcel;
-import android.os.Parcelable;
-import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
 import androidx.media3.common.util.UnstableApi;
 import androidx.media3.common.util.Util;
 
-/** A Label, as defined by ISO 23009-1, 4th edition, 5.3.7.2. */
+/** A label for a {@link Format}. */
 @UnstableApi
-public class Label implements Parcelable, Bundleable {
-  /** Declares the language code(s) for this Label. */
-  @Nullable public final String lang;
+public class Label {
+  /**
+   * The language of this label, as an IETF BCP 47 conformant tag, or null if unknown or not
+   * applicable.
+   */
+  @Nullable public final String language;
 
-  /** The value for this Label. */
+  /** The value for this label. */
   public final String value;
 
   /**
-   * @param lang The lang code.
-   * @param value The value.
+   * Creates a label.
+   *
+   * @param language The language of this label, as an IETF BCP 47 conformant tag, or null if
+   *     unknown or not applicable.
+   * @param value The label value.
    */
-  public Label(@Nullable String lang, String value) {
-    this.lang = lang;
+  public Label(@Nullable String language, String value) {
+    this.language = Util.normalizeLanguageCode(language);
     this.value = value;
   }
 
-  /* package */ Label(Parcel in) {
-    lang = in.readString();
-    value = in.readString();
-  }
-
   @Override
   public boolean equals(@Nullable Object o) {
-    if (this == o) return true;
-    if (o == null || getClass() != o.getClass()) return false;
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
     Label label = (Label) o;
-    return Util.areEqual(lang, label.lang) && Util.areEqual(value, label.value);
+    return Util.areEqual(language, label.language) && Util.areEqual(value, label.value);
   }
 
   @Override
   public int hashCode() {
     int result = value.hashCode();
-    result = 31 * result + (lang != null ? lang.hashCode() : 0);
+    result = 31 * result + (language != null ? language.hashCode() : 0);
     return result;
   }
 
-  @Override
-  public int describeContents() {
-    return 0;
-  }
-
-  @Override
-  public void writeToParcel(@NonNull Parcel dest, int flags) {
-    dest.writeString(lang);
-    dest.writeString(value);
-  }
-
-  public static final Parcelable.Creator<Label> CREATOR =
-      new Parcelable.Creator<Label>() {
-
-        @Override
-        public Label createFromParcel(Parcel in) {
-          return new Label(in);
-        }
-
-        @Override
-        public Label[] newArray(int size) {
-          return new Label[size];
-        }
-      };
-
-  private static final String FIELD_LANG_INDEX = Util.intToStringMaxRadix(0);
+  private static final String FIELD_LANGUAGE_INDEX = Util.intToStringMaxRadix(0);
   private static final String FIELD_VALUE_INDEX = Util.intToStringMaxRadix(1);
 
-  @Override
+  /** Serializes this instance to a {@link Bundle}. */
   public Bundle toBundle() {
     Bundle bundle = new Bundle();
-    bundle.putString(FIELD_LANG_INDEX, lang);
+    if (language != null) {
+      bundle.putString(FIELD_LANGUAGE_INDEX, language);
+    }
     bundle.putString(FIELD_VALUE_INDEX, value);
     return bundle;
   }
 
-  /**
-   * Constructs an instance of {@link Label} from a {@link Bundle} produced by {@link #toBundle()}.
-   */
+  /** Deserializes an instance from a {@link Bundle} produced by {@link #toBundle()}. */
   public static Label fromBundle(Bundle bundle) {
     return new Label(
-        bundle.getString(FIELD_LANG_INDEX), checkNotNull(bundle.getString(FIELD_VALUE_INDEX)));
+        bundle.getString(FIELD_LANGUAGE_INDEX), checkNotNull(bundle.getString(FIELD_VALUE_INDEX)));
   }
 }
diff --git a/libraries/common/src/test/java/androidx/media3/common/FormatTest.java b/libraries/common/src/test/java/androidx/media3/common/FormatTest.java
index 40ab4fc560..471f0e6b41 100644
--- a/libraries/common/src/test/java/androidx/media3/common/FormatTest.java
+++ b/libraries/common/src/test/java/androidx/media3/common/FormatTest.java
@@ -20,10 +20,12 @@
 import static androidx.media3.common.MimeTypes.VIDEO_WEBM;
 import static androidx.media3.test.utils.TestUtil.buildTestData;
 import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertThrows;
 
 import android.os.Bundle;
 import androidx.media3.test.utils.FakeMetadataEntry;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.common.collect.ImmutableList;
 import java.util.ArrayList;
 import java.util.List;
 import org.junit.Test;
@@ -59,44 +61,58 @@ public void roundTripViaBundle_excludeMetadata_hasMetadataExcluded() {
 
   @Test
   public void formatBuild_withLabelAndWithoutLabels_labelIsInLabels() {
-    Format format = createTestFormat();
-    format = format.buildUpon().setLabels(null).build();
+    Format format = new Format.Builder().setLabel("label").setLabels(ImmutableList.of()).build();
 
-    assertThat(format.labels.size()).isEqualTo(1);
-    assertThat(format.labels.get(0).value).isEqualTo(format.label);
+    assertThat(format.label).isEqualTo("label");
+    assertThat(format.labels).hasSize(1);
+    assertThat(format.labels.get(0).value).isEqualTo("label");
   }
 
   @Test
   public void formatBuild_withLabelsAndLanguageMatchingAndWithoutLabel_theLanguageMatchIsInLabel() {
-    Format format = createTestFormat();
-    format.labels.add(new Label("language", "matchingLabel"));
-    format = format.buildUpon().setLabel(null).build();
+    Format format =
+        new Format.Builder()
+            .setLabel(null)
+            .setLabels(
+                ImmutableList.of(
+                    new Label("en", "nonDefaultLabel"), new Label("zh", "matchingLabel")))
+            .setLanguage("zh")
+            .build();
 
     assertThat(format.label).isEqualTo("matchingLabel");
   }
 
   @Test
   public void formatBuild_withLabelsAndNoLanguageMatchingAndWithoutLabel_theFirstIsInLabel() {
-    Format format = createTestFormat();
-    format.labels.add(new Label("otherLanguage", "otherLabel"));
-    format = format.buildUpon().setLabel(null).build();
+    Format format =
+        new Format.Builder()
+            .setLabel(null)
+            .setLabels(
+                ImmutableList.of(new Label("fr", "firstLabel"), new Label("de", "secondLabel")))
+            .setLanguage("en")
+            .build();
 
-    assertThat(format.label).isEqualTo("label");
+    assertThat(format.label).isEqualTo("firstLabel");
   }
 
   @Test
-  public void formatBuild_withoutLabelsOrLabel_theyAreEmpty() {
+  public void formatBuild_withoutLabelsOrLabel_bothEmpty() {
     Format format = createTestFormat();
-    format = format.buildUpon().setLabel(null).setLabels(null).build();
+    format = format.buildUpon().setLabel(null).setLabels(ImmutableList.of()).build();
 
     assertThat(format.label).isEqualTo(null);
-    assertThat(format.labels.size()).isEqualTo(0);
+    assertThat(format.labels).isEmpty();
   }
 
-  @Test(expected = IllegalStateException.class)
-  public void formatBuild_withLabelAndLabelsSetButNotMatching_throwsException() {
-    Format format = createTestFormat();
-    format.buildUpon().setLabel("otherLabel").build();
+  @Test
+  public void formatBuild_withLabelAndLabelsSetButNoMatch_throwsException() {
+    assertThrows(
+        IllegalStateException.class,
+        () ->
+            new Format.Builder()
+                .setLabel("otherLabel")
+                .setLabels(ImmutableList.of(new Label("en", "label")))
+                .build());
   }
 
   private static Format createTestFormat() {
@@ -125,12 +141,10 @@ private static Format createTestFormat() {
             .setChromaBitdepth(11)
             .build();
 
-    List<Label> labels = new ArrayList<>();
-    labels.add(new Label("en", "label"));
     return new Format.Builder()
         .setId("id")
         .setLabel("label")
-        .setLabels(labels)
+        .setLabels(ImmutableList.of(new Label("en", "label")))
         .setLanguage("language")
         .setSelectionFlags(C.SELECTION_FLAG_DEFAULT)
         .setRoleFlags(C.ROLE_FLAG_MAIN)
diff --git a/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/manifest/DashManifestParser.java b/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/manifest/DashManifestParser.java
index 9e011d6bae..4fa9eea43c 100644
--- a/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/manifest/DashManifestParser.java
+++ b/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/manifest/DashManifestParser.java
@@ -864,10 +864,11 @@ protected Representation buildRepresentation(
       ArrayList<SchemeData> extraDrmSchemeDatas,
       ArrayList<Descriptor> extraInbandEventStreams) {
     Format.Builder formatBuilder = representationInfo.format.buildUpon();
-    if (label != null) {
+    if (label != null && labels.isEmpty()) {
       formatBuilder.setLabel(label);
+    } else {
+      formatBuilder.setLabels(labels);
     }
-    formatBuilder.setLabels(labels);
     @Nullable String drmSchemeType = representationInfo.drmSchemeType;
     if (drmSchemeType == null) {
       drmSchemeType = extraDrmSchemeType;
diff --git a/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/manifest/DashManifestParserTest.java b/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/manifest/DashManifestParserTest.java
index 3d2c806882..af2892c814 100644
--- a/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/manifest/DashManifestParserTest.java
+++ b/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/manifest/DashManifestParserTest.java
@@ -281,7 +281,10 @@ public void parseMediaPresentationDescription_labels() throws IOException {
 
     assertThat(adaptationSets.get(0).representations.get(0).format.label).isEqualTo("audio label");
     assertThat(adaptationSets.get(1).representations.get(0).format.label).isEqualTo("video label");
-    assertThat(adaptationSets.get(0).representations.get(0).format.labels.size()).isEqualTo(0);
+    assertThat(adaptationSets.get(0).representations.get(0).format.labels).hasSize(1);
+    assertThat(adaptationSets.get(0).representations.get(0).format.labels.get(0).value)
+        .isEqualTo("audio label");
+    assertThat(adaptationSets.get(1).representations.get(0).format.labels).hasSize(1);
     assertThat(adaptationSets.get(1).representations.get(0).format.labels.get(0).value)
         .isEqualTo("video label");
   }
@@ -441,7 +444,7 @@ public void parseLabel() throws Exception {
     xpp.next();
 
     Label label = parser.parseLabel(xpp);
-    assertThat(label.lang).isEqualTo("en");
+    assertThat(label.language).isEqualTo("en");
     assertThat(label.value).isEqualTo("test label");
     assertNextTag(xpp);
   }
@@ -454,7 +457,7 @@ public void parseLabel_noLang() throws Exception {
     xpp.next();
 
     Label label = parser.parseLabel(xpp);
-    assertThat(label.lang).isEqualTo(null);
+    assertThat(label.language).isEqualTo(null);
     assertNextTag(xpp);
   }
 
@@ -466,7 +469,8 @@ public void parseLabel_noText() throws Exception {
     xpp.next();
 
     Label label = parser.parseLabel(xpp);
-    assertThat(label.value).isEqualTo("");
+    assertThat(label.value).isNotNull();
+    assertThat(label.value).isEmpty();
     assertNextTag(xpp);
   }
 
diff --git a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaPeriod.java b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaPeriod.java
index 6ef59444ac..cfb7e82525 100644
--- a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaPeriod.java
+++ b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaPeriod.java
@@ -877,7 +877,7 @@ private static Format deriveAudioFormat(
     int roleFlags = 0;
     @Nullable String language = null;
     @Nullable String label = null;
-    @Nullable List<Label> labels = null;
+    List<Label> labels = ImmutableList.of();
     if (mediaTagFormat != null) {
       codecs = mediaTagFormat.codecs;
       metadata = mediaTagFormat.metadata;
diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/DumpableFormat.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/DumpableFormat.java
index f546e67e62..59cd6af726 100644
--- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/DumpableFormat.java
+++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/DumpableFormat.java
@@ -104,7 +104,7 @@ public void dump(Dumper dumper) {
     if (!format.labels.isEmpty()) {
       dumper.startBlock("labels");
       for (int i = 0; i < format.labels.size(); i++) {
-        String lang = format.labels.get(i).lang;
+        String lang = format.labels.get(i).language;
         if (lang != null) {
           dumper.add("lang", lang);
         }