Incorporate JsonPath into exception messages

core/api/kotlinx-serialization-core.api

@@ -339,7 +339,7 @@ public abstract class kotlinx/serialization/encoding/AbstractDecoder : kotlinx/s
 	public fun decodeFloat ()F
 	public final fun decodeFloatElement (Lkotlinx/serialization/descriptors/SerialDescriptor;I)F
 	public fun decodeInline (Lkotlinx/serialization/descriptors/SerialDescriptor;)Lkotlinx/serialization/encoding/Decoder;
-	public final fun decodeInlineElement (Lkotlinx/serialization/descriptors/SerialDescriptor;I)Lkotlinx/serialization/encoding/Decoder;
+	public fun decodeInlineElement (Lkotlinx/serialization/descriptors/SerialDescriptor;I)Lkotlinx/serialization/encoding/Decoder;
 	public fun decodeInt ()I
 	public final fun decodeIntElement (Lkotlinx/serialization/descriptors/SerialDescriptor;I)I
 	public fun decodeLong ()J
@@ -349,7 +349,7 @@ public abstract class kotlinx/serialization/encoding/AbstractDecoder : kotlinx/s
 	public final fun decodeNullableSerializableElement (Lkotlinx/serialization/descriptors/SerialDescriptor;ILkotlinx/serialization/DeserializationStrategy;Ljava/lang/Object;)Ljava/lang/Object;
 	public fun decodeNullableSerializableValue (Lkotlinx/serialization/DeserializationStrategy;)Ljava/lang/Object;
 	public fun decodeSequentially ()Z
-	public final fun decodeSerializableElement (Lkotlinx/serialization/descriptors/SerialDescriptor;ILkotlinx/serialization/DeserializationStrategy;Ljava/lang/Object;)Ljava/lang/Object;
+	public fun decodeSerializableElement (Lkotlinx/serialization/descriptors/SerialDescriptor;ILkotlinx/serialization/DeserializationStrategy;Ljava/lang/Object;)Ljava/lang/Object;
 	public fun decodeSerializableValue (Lkotlinx/serialization/DeserializationStrategy;)Ljava/lang/Object;
 	public fun decodeSerializableValue (Lkotlinx/serialization/DeserializationStrategy;Ljava/lang/Object;)Ljava/lang/Object;
 	public static synthetic fun decodeSerializableValue$default (Lkotlinx/serialization/encoding/AbstractDecoder;Lkotlinx/serialization/DeserializationStrategy;Ljava/lang/Object;ILjava/lang/Object;)Ljava/lang/Object;

core/commonMain/src/kotlinx/serialization/encoding/AbstractDecoder.kt

@@ -57,12 +57,12 @@ public abstract class AbstractDecoder : Decoder, CompositeDecoder {
     final override fun decodeCharElement(descriptor: SerialDescriptor, index: Int): Char = decodeChar()
     final override fun decodeStringElement(descriptor: SerialDescriptor, index: Int): String = decodeString()
 
-    final override fun decodeInlineElement(
+    override fun decodeInlineElement(
         descriptor: SerialDescriptor,
         index: Int
     ): Decoder = decodeInline(descriptor.getElementDescriptor(index))
 
-    final override fun <T> decodeSerializableElement(
+    override fun <T> decodeSerializableElement(
         descriptor: SerialDescriptor,
         index: Int,
         deserializer: DeserializationStrategy<T>,

docs/basic-serialization.md

@@ -297,7 +297,7 @@ fun main() {
 It produces the exception:
 
 ```text
-Exception in thread "main" kotlinx.serialization.MissingFieldException: Field 'language' is required for type with serial name 'example.exampleClasses04.Project', but it was missing
+Exception in thread "main" kotlinx.serialization.MissingFieldException: Field 'language' is required for type with serial name 'example.exampleClasses04.Project', but it was missing at path: $
 ```   
 
 <!--- TEST LINES_START -->
@@ -383,7 +383,7 @@ fun main() {
 We get the following exception.
 
 ```text
-Exception in thread "main" kotlinx.serialization.MissingFieldException: Field 'language' is required for type with serial name 'example.exampleClasses07.Project', but it was missing
+Exception in thread "main" kotlinx.serialization.MissingFieldException: Field 'language' is required for type with serial name 'example.exampleClasses07.Project', but it was missing at path: $
 ```   
 
 <!--- TEST LINES_START -->
@@ -411,7 +411,7 @@ Attempts to explicitly specify its value in the serial format, even if the speci
 value is equal to the default one, produces the following exception.
 
 ```text
-Exception in thread "main" kotlinx.serialization.json.internal.JsonDecodingException: Unexpected JSON token at offset 42: Encountered an unknown key 'language'.
+Exception in thread "main" kotlinx.serialization.json.internal.JsonDecodingException: Unexpected JSON token at offset 42: Encountered an unknown key 'language' at path: $.name
 Use 'ignoreUnknownKeys = true' in 'Json {}' builder to ignore unknown keys.
 ```   
 
@@ -493,7 +493,7 @@ Even though the `language` property has a default value, it is still an error to
 the `null` value to it.
 
 ```text
-Exception in thread "main" kotlinx.serialization.json.internal.JsonDecodingException: Unexpected JSON token at offset 52: Expected string literal but 'null' literal was found.
+Exception in thread "main" kotlinx.serialization.json.internal.JsonDecodingException: Unexpected JSON token at offset 52: Expected string literal but 'null' literal was found at path: $.language
 Use 'coerceInputValues = true' in 'Json {}` builder to coerce nulls to default values.
 ```                    
 

formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonExceptions.kt

@@ -38,20 +38,29 @@ internal fun InvalidFloatingPointEncoded(value: Number, output: String) = JsonEn
             "Current output: ${output.minify()}"
 )
 
-internal fun InvalidFloatingPointEncoded(value: Number, key: String, output: String) =
-    JsonEncodingException(unexpectedFpErrorMessage(value, key, output))
-
-internal fun InvalidFloatingPointDecoded(value: Number, key: String, output: String) =
-    JsonDecodingException(-1, unexpectedFpErrorMessage(value, key, output))
 
 // Extension on JSON reader and fail immediately
 internal fun AbstractJsonLexer.throwInvalidFloatingPointDecoded(result: Number): Nothing {
     fail("Unexpected special floating-point value $result. By default, " +
-            "non-finite floating point values are prohibited because they do not conform JSON specification. " +
-            specialFlowingValuesHint
-    )
+            "non-finite floating point values are prohibited because they do not conform JSON specification",
+        hint = specialFlowingValuesHint)
 }
 
+@OptIn(ExperimentalSerializationApi::class)
+internal fun InvalidKeyKindException(keyDescriptor: SerialDescriptor) = JsonEncodingException(
+    "Value of type '${keyDescriptor.serialName}' can't be used in JSON as a key in the map. " +
+            "It should have either primitive or enum kind, but its kind is '${keyDescriptor.kind}'.\n" +
+            allowStructuredMapKeysHint
+)
+
+// Exceptions for tree-based decoder
+
+internal fun InvalidFloatingPointEncoded(value: Number, key: String, output: String) =
+    JsonEncodingException(unexpectedFpErrorMessage(value, key, output))
+
+internal fun InvalidFloatingPointDecoded(value: Number, key: String, output: String) =
+    JsonDecodingException(-1, unexpectedFpErrorMessage(value, key, output))
+
 private fun unexpectedFpErrorMessage(value: Number, key: String, output: String): String {
     return "Unexpected special floating-point value $value with key $key. By default, " +
             "non-finite floating point values are prohibited because they do not conform JSON specification. " +
@@ -66,13 +75,6 @@ internal fun UnknownKeyException(key: String, input: String) = JsonDecodingExcep
             "Current input: ${input.minify()}"
 )
 
-@OptIn(ExperimentalSerializationApi::class)
-internal fun InvalidKeyKindException(keyDescriptor: SerialDescriptor) = JsonEncodingException(
-    "Value of type '${keyDescriptor.serialName}' can't be used in JSON as a key in the map. " +
-            "It should have either primitive or enum kind, but its kind is '${keyDescriptor.kind}'.\n" +
-            allowStructuredMapKeysHint
-)
-
 private fun CharSequence.minify(offset: Int = -1): CharSequence {
     if (length < 200) return this
     if (offset == -1) {

formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonNamesMap.kt

@@ -57,10 +57,10 @@ internal fun SerialDescriptor.getJsonNameIndex(json: Json, name: String): Int {
  * Throws on [CompositeDecoder.UNKNOWN_NAME]
  */
 @OptIn(ExperimentalSerializationApi::class)
-internal fun SerialDescriptor.getJsonNameIndexOrThrow(json: Json, name: String): Int {
+internal fun SerialDescriptor.getJsonNameIndexOrThrow(json: Json, name: String, suffix: String = ""): Int {
     val index = getJsonNameIndex(json, name)
     if (index == CompositeDecoder.UNKNOWN_NAME)
-        throw SerializationException("$serialName does not contain element with name '$name'")
+        throw SerializationException("$serialName does not contain element with name '$name'$suffix")
     return index
 }
 

formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonPath.kt

@@ -0,0 +1,141 @@
+package kotlinx.serialization.json.internal
+
+import kotlinx.serialization.*
+import kotlinx.serialization.descriptors.*
+import kotlinx.serialization.internal.*
+
+/**
+ * Internal representation of the current JSON path.
+ * It is stored as the array of serial descriptors (for regular classes)
+ * and `Any?` in case of Map keys.
+ *
+ * Example of the state when decoding the list
+ * ```
+ * class Foo(val a: Int, val l: List<String>)
+ *
+ * // {"l": ["a", "b", "c"] }
+ *
+ * Current path when decoding array elements:
+ * Foo.descriptor, List(String).descriptor
+ * 1 (index of the 'l'), 2 (index of currently being decoded "c")
+ * ```
+ */
+internal class JsonPath {
+
+    // Tombstone indicates that we are within a map, but the map key is currently being decoded.
+    // It is also used to overwrite a previous map key to avoid memory leaks and misattribution.
+    object Tombstone
+
+    /*
+     * Serial descriptor, map key or the tombstone for map key
+     */
+    private var currentObjectPath = arrayOfNulls<Any?>(8)
+    /*
+     * Index is a small state-machine used to determine the state of the path:
+     * >=0 -> index of the element being decoded with the outer class currentObjectPath[currentDepth]
+     * -1 -> nested elements are not yet decoded
+     * -2 -> the map is being decoded and both its descriptor AND the last key were added to the path.
+     *
+     * -2 is effectively required to specify that two slots has been claimed and both should be
+     * cleaned up when the decoding is done.
+     * The cleanup is essential in order to avoid memory leaks for huge strings and structured keys.
+     */
+    private var indicies = IntArray(8) { -1 }
+    private var currentDepth = -1
+
+    // Invoked when class is started being decoded
+    fun pushDescriptor(sd: SerialDescriptor) {
+        val depth = ++currentDepth
+        if (depth == currentObjectPath.size) {
+            resize()
+        }
+        currentObjectPath[depth] = sd
+    }
+
+    // Invoked when index-th element of the current descriptor is being decoded
+    fun updateDescriptorIndex(index: Int) {
+        indicies[currentDepth] = index
+    }
+
+    /*
+     * For maps we cannot use indicies and should use the key as an element of the path instead.
+     * The key can be even an object (e.g. in a case of 'allowStructuredMapKeys') where
+     * 'toString' is way too heavy or have side-effects.
+     * For that we are storing the key instead.
+     */
+    fun updateCurrentMapKey(key: Any?) {
+        // idx != -2 -> this is the very first key being added
+        if (indicies[currentDepth] != -2 && ++currentDepth == currentObjectPath.size) {
+            resize()
+        }
+        currentObjectPath[currentDepth] = key
+        indicies[currentDepth] = -2
+    }
+
+    /** Used to indicate that we are in the process of decoding the key itself and can't specify it in path */
+    fun resetCurrentMapKey() {
+        if (indicies[currentDepth] == -2) {
+            currentObjectPath[currentDepth] = Tombstone
+        }
+    }
+
+    fun popDescriptor() {
+        // When we are ending map, we pop the last key and the outer field as well
+        val depth = currentDepth
+        if (indicies[depth] == -2) {
+            indicies[depth] = -1
+            currentDepth--
+        }
+        // Guard against top-level maps
+        if (currentDepth != -1) {
+            // No need to clean idx up as it was already cleaned by updateDescriptorIndex(DECODE_DONE)
+            currentDepth--
+        }
+    }
+
+    @OptIn(ExperimentalSerializationApi::class)
+    fun getPath(): String {
+        return buildString {
+            append("$")
+            repeat(currentDepth + 1) {
+                val element = currentObjectPath[it]
+                if (element is SerialDescriptor) {
+                    if (element.kind == StructureKind.LIST) {
+                        if (indicies[it] != -1) {
+                            append("[")
+                            append(indicies[it])
+                            append("]")
+                        }
+                    } else {
+                        val idx = indicies[it]
+                        // If an actual element is being decoded
+                        if (idx >= 0) {
+                            append(".")
+                            append(element.getElementName(idx))
+                        }
+                    }
+                } else if (element !== Tombstone) {
+                    append("[")
+                    // All non-indicies should be properly quoted by JsonPath convention
+                    append("'")
+                    // Else -- map key
+                    append(element)
+                    append("'")
+                    append("]")
+                }
+            }
+        }
+    }
+
+
+    @OptIn(ExperimentalSerializationApi::class)
+    private fun prettyString(it: Any?) = (it as? SerialDescriptor)?.serialName ?: it.toString()
+
+    private fun resize() {
+        val newSize = currentDepth * 2
+        currentObjectPath = currentObjectPath.copyOf(newSize)
+        indicies = indicies.copyOf(newSize)
+    }
+
+    override fun toString(): String = getPath()
+}

formats/json/commonMain/src/kotlinx/serialization/json/internal/StreamingJsonDecoder.kt

@@ -32,12 +32,18 @@ internal open class StreamingJsonDecoder(
 
     override fun decodeJsonElement(): JsonElement = JsonTreeReader(json.configuration, lexer).read()
 
+    @Suppress("INVISIBLE_MEMBER", "INVISIBLE_REFERENCE")
     override fun <T> decodeSerializableValue(deserializer: DeserializationStrategy<T>): T {
-        return decodeSerializableValuePolymorphic(deserializer)
+        try {
+            return decodeSerializableValuePolymorphic(deserializer)
+        } catch (e: MissingFieldException) {
+            throw MissingFieldException(e.message + " at path: " + lexer.path.getPath(), e)
+        }
     }
 
     override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder {
         val newMode = json.switchMode(descriptor)
+        lexer.path.pushDescriptor(descriptor)
         lexer.consumeNextToken(newMode.begin)
         checkLeadingComma()
         return when (newMode) {
@@ -63,7 +69,10 @@ internal open class StreamingJsonDecoder(
         if (json.configuration.ignoreUnknownKeys && descriptor.elementsCount == 0) {
             skipLeftoverElements(descriptor)
         }
+        // First consume the object so we know it's correct
         lexer.consumeNextToken(mode.end)
+        // Then cleanup the path
+        lexer.path.popDescriptor()
     }
 
     private fun skipLeftoverElements(descriptor: SerialDescriptor) {
@@ -87,12 +96,37 @@ internal open class StreamingJsonDecoder(
         }
     }
 
+    override fun <T> decodeSerializableElement(
+        descriptor: SerialDescriptor,
+        index: Int,
+        deserializer: DeserializationStrategy<T>,
+        previousValue: T?
+    ): T {
+        val isMapKey = mode == WriteMode.MAP && index and 1 == 0
+        // Reset previous key
+        if (isMapKey) {
+            lexer.path.resetCurrentMapKey()
+        }
+        // Deserialize the key
+        val value = super.decodeSerializableElement(descriptor, index, deserializer, previousValue)
+        // Put the key to the path
+        if (isMapKey) {
+            lexer.path.updateCurrentMapKey(value)
+        }
+        return value
+    }
+
     override fun decodeElementIndex(descriptor: SerialDescriptor): Int {
-        return when (mode) {
+        val index = when (mode) {
             WriteMode.OBJ -> decodeObjectIndex(descriptor)
             WriteMode.MAP -> decodeMapIndex()
             else -> decodeListIndex() // Both for LIST and default polymorphic
         }
+        // The element of the next index that will be decoded
+        if (mode != WriteMode.MAP) {
+            lexer.path.updateDescriptorIndex(index)
+        }
+        return index
     }
 
     private fun decodeMapIndex(): Int {
@@ -262,7 +296,7 @@ internal open class StreamingJsonDecoder(
         else super.decodeInline(inlineDescriptor)
 
     override fun decodeEnum(enumDescriptor: SerialDescriptor): Int {
-        return enumDescriptor.getJsonNameIndexOrThrow(json, decodeString())
+        return enumDescriptor.getJsonNameIndexOrThrow(json, decodeString(), " at path " + lexer.path.getPath())
     }
 }