From 189eb6e38e78a06d86292b5d503de12fe2bebe38 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Sun, 17 Jul 2022 13:35:34 -0700 Subject: [PATCH] DEP: Finalize ragged array creation deprecation This finalizes all DeprcationWarnings related to the creation of object arrays from nested lists enforcing that users use `dtype=object` if this is intentional. --- .../upcoming_changes/22004.expired.rst | 2 + numpy/core/src/multiarray/array_coercion.c | 92 +------------------ numpy/core/tests/test_array_coercion.py | 9 +- numpy/core/tests/test_deprecations.py | 42 --------- numpy/core/tests/test_multiarray.py | 9 +- 5 files changed, 14 insertions(+), 140 deletions(-) create mode 100644 doc/release/upcoming_changes/22004.expired.rst diff --git a/doc/release/upcoming_changes/22004.expired.rst b/doc/release/upcoming_changes/22004.expired.rst new file mode 100644 index 000000000000..e07dffa84dbe --- /dev/null +++ b/doc/release/upcoming_changes/22004.expired.rst @@ -0,0 +1,2 @@ +* Ragged array creation will now always raise a ``ValueError`` unless + ``dtype=object`` is passed. This includes very deeply nested sequences. diff --git a/numpy/core/src/multiarray/array_coercion.c b/numpy/core/src/multiarray/array_coercion.c index e703e73820b4..117ce1805b62 100644 --- a/numpy/core/src/multiarray/array_coercion.c +++ b/numpy/core/src/multiarray/array_coercion.c @@ -1307,95 +1307,10 @@ PyArray_DiscoverDTypeAndShape( /* Handle reaching the maximum depth differently: */ int too_deep = ndim == max_dims; - if (fixed_DType == NULL) { - /* This is discovered as object, but deprecated */ - static PyObject *visibleDeprecationWarning = NULL; - npy_cache_import( - "numpy", "VisibleDeprecationWarning", - &visibleDeprecationWarning); - if (visibleDeprecationWarning == NULL) { - goto fail; - } - if (!too_deep) { - /* NumPy 1.19, 2019-11-01 */ - if (PyErr_WarnEx(visibleDeprecationWarning, - "Creating an ndarray from ragged nested sequences (which " - "is a list-or-tuple of lists-or-tuples-or ndarrays with " - "different lengths or shapes) is deprecated. If you " - "meant to do this, you must specify 'dtype=object' " - "when creating the ndarray.", 1) < 0) { - goto fail; - } - } - else { - /* NumPy 1.20, 2020-05-08 */ - /* Note, max_dims should normally always be NPY_MAXDIMS here */ - if (PyErr_WarnFormat(visibleDeprecationWarning, 1, - "Creating an ndarray from nested sequences exceeding " - "the maximum number of dimensions of %d is deprecated. " - "If you mean to do this, you must specify " - "'dtype=object' when creating the ndarray.", - max_dims) < 0) { - goto fail; - } - } - /* Ensure that ragged arrays always return object dtype */ - Py_XSETREF(*out_descr, PyArray_DescrFromType(NPY_OBJECT)); - } - else if (fixed_DType->type_num != NPY_OBJECT) { + if (fixed_DType == NULL || fixed_DType->type_num != NPY_OBJECT) { /* Only object DType supports ragged cases unify error */ - /* - * We used to let certain ragged arrays pass if they also - * support e.g. conversion using `float(arr)`, which currently - * works for arrays with only one element. - * Thus we catch at least most of such cases here and give a - * DeprecationWarning instead of an error. - * Note that some of these will actually error later on when - * attempting to do the actual assign. - */ - int deprecate_single_element_ragged = 0; - coercion_cache_obj *current = *coercion_cache_head; - while (current != NULL) { - if (current->sequence) { - if (current->depth == ndim) { - /* - * Assume that only array-likes will allow the deprecated - * behaviour - */ - deprecate_single_element_ragged = 0; - break; - } - /* check next converted sequence/array-like */ - current = current->next; - continue; - } - PyArrayObject *arr = (PyArrayObject *)(current->arr_or_sequence); - assert(PyArray_NDIM(arr) + current->depth >= ndim); - if (PyArray_NDIM(arr) != ndim - current->depth) { - /* This array is not compatible with the final shape */ - if (PyArray_SIZE(arr) != 1) { - deprecate_single_element_ragged = 0; - break; - } - deprecate_single_element_ragged = 1; - } - current = current->next; - } - - if (deprecate_single_element_ragged) { - /* Deprecated 2020-07-24, NumPy 1.20 */ - if (DEPRECATE( - "setting an array element with a sequence. " - "This was supported in some cases where the elements " - "are arrays with a single element. For example " - "`np.array([1, np.array([2])], dtype=int)`. " - "In the future this will raise the same ValueError as " - "`np.array([1, [2]], dtype=int)`.") < 0) { - goto fail; - } - } - else if (!too_deep) { + if (!too_deep) { PyObject *shape = PyArray_IntTupleFromIntp(ndim, out_shape); PyErr_Format(PyExc_ValueError, "setting an array element with a sequence. The " @@ -1404,7 +1319,6 @@ PyArray_DiscoverDTypeAndShape( "%R + inhomogeneous part.", ndim, shape); Py_DECREF(shape); - goto fail; } else { PyErr_Format(PyExc_ValueError, @@ -1412,8 +1326,8 @@ PyArray_DiscoverDTypeAndShape( "requested array would exceed the maximum number of " "dimension of %d.", max_dims); - goto fail; } + goto fail; } /* diff --git a/numpy/core/tests/test_array_coercion.py b/numpy/core/tests/test_array_coercion.py index ed3ef7e678ec..6abe942bd480 100644 --- a/numpy/core/tests/test_array_coercion.py +++ b/numpy/core/tests/test_array_coercion.py @@ -490,9 +490,10 @@ def test_nested_simple(self): with pytest.raises(ValueError): np.array([nested], dtype="float64") - # We discover object automatically at this time: - with assert_warns(np.VisibleDeprecationWarning): - arr = np.array([nested]) + with pytest.raises(ValueError, match=".*would exceed the maximum"): + np.array([nested]) # user must ask for `object` explicitly + + arr = np.array([nested], dtype=object) assert arr.dtype == np.dtype("O") assert arr.shape == (1,) * np.MAXDIMS assert arr.item() is initial @@ -523,7 +524,7 @@ def test_nested_arraylikes(self, arraylike): for i in range(np.MAXDIMS - 1): nested = [nested] - with pytest.warns(DeprecationWarning): + with pytest.raises(ValueError, match=".*would exceed the maximum"): # It will refuse to assign the array into np.array(nested, dtype="float64") diff --git a/numpy/core/tests/test_deprecations.py b/numpy/core/tests/test_deprecations.py index 2255cb2a3835..5c13fcd4fbc7 100644 --- a/numpy/core/tests/test_deprecations.py +++ b/numpy/core/tests/test_deprecations.py @@ -475,34 +475,6 @@ def test_zerod(self): self.assert_deprecated(lambda: np.nonzero(np.array(1))) -def test_deprecate_ragged_arrays(): - # 2019-11-29 1.19.0 - # - # NEP 34 deprecated automatic object dtype when creating ragged - # arrays. Also see the "ragged" tests in `test_multiarray` - # - # emits a VisibleDeprecationWarning - arg = [1, [2, 3]] - with assert_warns(np.VisibleDeprecationWarning): - np.array(arg) - - -class TestTooDeepDeprecation(_VisibleDeprecationTestCase): - # NumPy 1.20, 2020-05-08 - # This is a bit similar to the above ragged array deprecation case. - message = re.escape("Creating an ndarray from nested sequences exceeding") - - def test_deprecation(self): - nested = [1] - for i in range(np.MAXDIMS - 1): - nested = [nested] - self.assert_not_deprecated(np.array, args=(nested,)) - self.assert_not_deprecated(np.array, - args=(nested,), kwargs=dict(dtype=object)) - - self.assert_deprecated(np.array, args=([nested],)) - - class TestToString(_DeprecationTestCase): # 2020-03-06 1.19.0 message = re.escape("tostring() is deprecated. Use tobytes() instead.") @@ -644,20 +616,6 @@ def test_deprecated(self): self.assert_not_deprecated(np.add.outer, args=(arr, arr)) -class TestRaggedArray(_DeprecationTestCase): - # 2020-07-24, NumPy 1.20.0 - message = "setting an array element with a sequence" - - def test_deprecated(self): - arr = np.ones((1, 1)) - # Deprecated if the array is a leave node: - self.assert_deprecated(lambda: np.array([arr, 0], dtype=np.float64)) - self.assert_deprecated(lambda: np.array([0, arr], dtype=np.float64)) - # And when it is an assignment into a lower dimensional subarray: - self.assert_deprecated(lambda: np.array([arr, [0]], dtype=np.float64)) - self.assert_deprecated(lambda: np.array([[0], arr], dtype=np.float64)) - - class FlatteningConcatenateUnsafeCast(_DeprecationTestCase): # NumPy 1.20, 2020-09-03 message = "concatenate with `axis=None` will use same-kind casting" diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 9f6023c07988..a36aca361b5e 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -1118,12 +1118,11 @@ def test_array_too_big(self): shape=(max_bytes//itemsize + 1,), dtype=dtype) def _ragged_creation(self, seq): - # without dtype=object, the ragged object should raise - with assert_warns(np.VisibleDeprecationWarning): + # without dtype=object, the ragged object raises + with pytest.raises(ValueError, match=".*detected shape was"): a = np.array(seq) - b = np.array(seq, dtype=object) - assert_equal(a, b) - return b + + return np.array(seq, dtype=object) def test_ragged_ndim_object(self): # Lists of mismatching depths are treated as object arrays